Browse Source

initial commit

master
boB Rudis 3 years ago
parent
commit
8f387319b2
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
  1. 1
      .Rbuildignore
  2. 13
      DESCRIPTION
  3. 2
      LICENSE
  4. 21
      LICENSE.md
  5. 6
      NAMESPACE
  6. 100
      R/brim-main.R
  7. 7
      R/brimr-package.R
  8. 15
      README.Rmd
  9. 311
      README.md
  10. 16
      man/brim_ast.Rd
  11. 13
      man/brim_host.Rd
  12. 18
      man/brim_search.Rd
  13. 18
      man/brim_search_raw.Rd
  14. 14
      man/brim_spaces.Rd
  15. 6
      man/brimr.Rd

1
.Rbuildignore

@ -19,3 +19,4 @@
^CRAN-RELEASE$
^appveyor\.yml$
^tools$
^LICENSE\.md$

13
DESCRIPTION

@ -1,6 +1,6 @@
Package: brimr
Type: Package
Title: brimr title goes here otherwise CRAN checks fail
Title: Tools to Work with Brim and zqd
Version: 0.1.0
Date: 2021-02-28
Authors@R: c(
@ -8,17 +8,20 @@ Authors@R: c(
comment = c(ORCID = "0000-0001-5670-2640"))
)
Maintainer: Bob Rudis <bob@rud.is>
Description: A good description goes here otherwise CRAN checks fail.
Description: Brim (<https://github.com/brimsec/brim>) enables efficient query
operations on large packqet captures and log sources, such as Zeek. Tools
are provided to with with Brim components, including the Brim zqd query back-end.
URL: https://git.rud.is/hrbrmstr/brimr
BugReports: https://git.rud.is/hrbrmstr/brimr/issues
Encoding: UTF-8
License: AGPL
License: MIT + file LICENSE
Suggests:
covr, tinytest
Depends:
R (>= 3.5.0)
R (>= 3.6.0)
Imports:
httr,
jsonlite
jsonlite,
stringi
Roxygen: list(markdown = TRUE)
RoxygenNote: 7.1.1

2
LICENSE

@ -0,0 +1,2 @@
YEAR: 2021
COPYRIGHT HOLDER: Bob Rudis

21
LICENSE.md

@ -0,0 +1,21 @@
# MIT License
Copyright (c) 2021 Bob Rudis
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

6
NAMESPACE

@ -1,4 +1,10 @@
# Generated by roxygen2: do not edit by hand
export(brim_ast)
export(brim_host)
export(brim_search)
export(brim_search_raw)
export(brim_spaces)
import(httr)
importFrom(jsonlite,fromJSON)
importFrom(stringi,stri_split_lines)

100
R/brim-main.R

@ -0,0 +1,100 @@
#' Retrieve the Brim host URL
#'
#' Looks in the `BRIM_HOST` environment variable for the host
#' URL of the Brim instance to use for the API call. This
#' defaults to `http://127.0.0.1:9867`.
#'
#' @export
brim_host <- function() {
Sys.getenv("BRIM_HOST", unset = "http://127.0.0.1:9867")
}
#' Turn a Brim ZQL query into an abstract syntax tree
#'
#' @param zql the ZQL query
#' @param host see [brim_host()]
#' @export
brim_ast <- function(zql, host = brim_host()) {
httr::POST(
url = sprintf("%s/ast", host),
httr::content_type_json(),
encode = "json",
body = list(
zql = zql
)
) -> res
httr::stop_for_status(res)
invisible(httr::content(res, "text", encoding = "UTF-8"))
}
#' Retrieve active Brim spaces from the specified Brim instance
#'
#' @param host see [brim_host()]
#' @export
brim_spaces <- function(host = brim_host()) {
httr::GET(
url = sprintf("%s/space", host),
) -> res
httr::stop_for_status(res)
out <- httr::content(res, as = "text", encoding = "UTF-8")
out <- jsonlite::fromJSON(out)
class(out) <- c("tbl_df", "tbl", "data.frame")
out
}
#' Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format
#'
#' @param space_name name of the Brim space to use as the search data source
#' @param zql the ZQL query
#' @param host see [brim_host()]
#' @export
brim_search_raw <- function(space_name, zql, host = brim_host()) {
available_spaces <- brim_spaces()
space_id <- available_spaces[available_spaces[["name"]] == space_name, "id", drop=TRUE]
x <- sprintf('{"dir":-1,"proc":%s,"space":"%s"}', trimws(brim_ast(zql)), space_id)
httr::POST(
url = "http://127.0.0.1:9867/search",
encode = "raw",
httr::content_type_json(),
query = list(
format = "zjson"
),
body = x
) -> res
httr::stop_for_status(res)
out <- httr::content(res, as = "text", encoding = "UTF-8")
out
}
#' Post a ZQL query to the given Brim instance and retrieve processed results
#'
#' @param space_name name of the Brim space to use as the search data source
#' @param zql the ZQL query
#' @param host see [brim_host()]
#' @export
brim_search <- function(space_name, zql, host = brim_host()) {
res <- brim_search_raw(space_name = space_name, zql = zql, host = host)
res <- stringi::stri_split_lines(res, omit_empty = TRUE)
res <- unlist(res)
res <- lapply(res, jsonlite::fromJSON)
invisible(res)
}

7
R/brimr-package.R

@ -1,9 +1,14 @@
#' ...
#' Tools to Work with Brim and zqd
#'
#' Brim (<https://github.com/brimsec/brim>) enables efficient query
#' operations on large packqet captures and log sources, such as Zeek. Tools
#' are provided to with with Brim components, including the Brim zqd query back-end.
#'
#' @md
#' @name brimr
#' @keywords internal
#' @author Bob Rudis (bob@@rud.is)
#' @import httr
#' @importFrom stringi stri_split_lines
#' @importFrom jsonlite fromJSON
"_PACKAGE"

15
README.Rmd

@ -33,12 +33,27 @@ hrbrpkghelpr::install_block()
```{r lib-ex}
library(brimr)
library(tibble)
# current version
packageVersion("brimr")
```
```{r ex-01}
brim_spaces()
zql <- '_path=conn | count() by id.orig_h, id.resp_h, id.resp_p | sort id.orig_h, id.resp_h, id.resp_p'
cat(jsonlite::toJSON(jsonlite::fromJSON(brim_ast(zql)), pretty = TRUE))
space <- "2021-02-17-Trickbot-gtag-rob13-infection-in-AD-environment.pcap"
r <- brim_search(space, zql)
str(r, 2)
```
## brimr Metrics
```{r cloc, echo=FALSE}

311
README.md

@ -0,0 +1,311 @@
[![Project Status: Active – The project has reached a stable, usable
state and is being actively
developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
[![Signed
by](https://img.shields.io/badge/Keybase-Verified-brightgreen.svg)](https://keybase.io/hrbrmstr)
![Signed commit
%](https://img.shields.io/badge/Signed_Commits-100%25-lightgrey.svg)
[![Linux build
Status](https://travis-ci.org/hrbrmstr/brimr.svg?branch=master)](https://travis-ci.org/hrbrmstr/brimr)
![Minimal R
Version](https://img.shields.io/badge/R%3E%3D-3.6.0-blue.svg)
![License](https://img.shields.io/badge/License-MIT-blue.svg)
# brimr
Tools to Work with Brim and zqd
## Description
Brim (<https://github.com/brimsec/brim>) enables efficient query
operations on large packqet captures and log sources, such as Zeek.
Tools are provided to with with Brim components, including the Brim zqd
query back-end.
## What’s Inside The Tin
The following functions are implemented:
- `brim_ast`: Turn a Brim ZQL query into an abstract syntax tree
- `brim_host`: Retrieve the Brim host URL
- `brim_search_raw`: Post a ZQL query to the given Brim instance and
retrieve results in raq ZJSON format
- `brim_search`: Post a ZQL query to the given Brim instance and
retrieve processed results
- `brim_spaces`: Retrieve active Brim spaces from the specified Brim
instance
## Installation
``` r
remotes::install_git("https://git.rud.is/hrbrmstr/brimr.git")
# or
remotes::install_gitlab("hrbrmstr/brimr")
# or
remotes::install_bitbucket("hrbrmstr/brimr")
```
NOTE: To use the ‘remotes’ install options you will need to have the
[{remotes} package](https://github.com/r-lib/remotes) installed.
## Usage
``` r
library(brimr)
library(tibble)
# current version
packageVersion("brimr")
## [1] '0.1.0'
```
``` r
brim_spaces()
## # A tibble: 1 x 4
## id name data_path storage_kind
## * <chr> <chr> <chr> <chr>
## 1 sp_1p6pwLgtsESYBT… 2021-02-17-Trickbot-gtag-rob13-i… file:///Users/hrbrmstr/Library/Application%20Suppor… filestore
zql <- '_path=conn | count() by id.orig_h, id.resp_h, id.resp_p | sort id.orig_h, id.resp_h, id.resp_p'
cat(jsonlite::toJSON(jsonlite::fromJSON(brim_ast(zql)), pretty = TRUE))
## {
## "op": ["SequentialProc"],
## "procs": [
## {
## "op": "FilterProc",
## "filter": {
## "op": "CompareField",
## "comparator": "=",
## "field": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "RootRecord"
## },
## "rhs": {
## "op": "Identifier",
## "name": "_path"
## }
## },
## "value": {
## "op": "Literal",
## "type": "string",
## "value": "conn"
## }
## },
## "keys": {},
## "reducers": {},
## "fields": {}
## },
## {
## "op": "GroupByProc",
## "filter": {
## "field": {
## "lhs": {},
## "rhs": {}
## },
## "value": {}
## },
## "limit": 0,
## "keys": [
## {
## "op": "Assignment",
## "rhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "RootRecord"
## },
## "rhs": {
## "op": "Identifier",
## "name": "id"
## }
## },
## "rhs": {
## "op": "Identifier",
## "name": "orig_h"
## }
## }
## },
## {
## "op": "Assignment",
## "rhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "RootRecord"
## },
## "rhs": {
## "op": "Identifier",
## "name": "id"
## }
## },
## "rhs": {
## "op": "Identifier",
## "name": "resp_h"
## }
## }
## },
## {
## "op": "Assignment",
## "rhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "RootRecord"
## },
## "rhs": {
## "op": "Identifier",
## "name": "id"
## }
## },
## "rhs": {
## "op": "Identifier",
## "name": "resp_p"
## }
## }
## }
## ],
## "reducers": [
## {
## "op": "Assignment",
## "rhs": {
## "op": "Reducer",
## "operator": "count"
## }
## }
## ],
## "fields": {}
## },
## {
## "op": "SortProc",
## "filter": {
## "field": {
## "lhs": {},
## "rhs": {}
## },
## "value": {}
## },
## "keys": {},
## "reducers": {},
## "fields": [
## {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "RootRecord"
## },
## "rhs": {
## "op": "Identifier",
## "name": "id"
## }
## },
## "rhs": {
## "op": "Identifier",
## "name": "orig_h"
## }
## },
## {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "RootRecord"
## },
## "rhs": {
## "op": "Identifier",
## "name": "id"
## }
## },
## "rhs": {
## "op": "Identifier",
## "name": "resp_h"
## }
## },
## {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "BinaryExpr",
## "operator": ".",
## "lhs": {
## "op": "RootRecord"
## },
## "rhs": {
## "op": "Identifier",
## "name": "id"
## }
## },
## "rhs": {
## "op": "Identifier",
## "name": "resp_p"
## }
## }
## ],
## "sortdir": 1,
## "nullsfirst": false
## }
## ]
## }
space <- "2021-02-17-Trickbot-gtag-rob13-infection-in-AD-environment.pcap"
r <- brim_search(space, zql)
str(r, 2)
## List of 5
## $ :List of 2
## ..$ type : chr "TaskStart"
## ..$ task_id: int 0
## $ :List of 3
## ..$ type : chr "SearchRecords"
## ..$ channel_id: int 0
## ..$ records :'data.frame': 74 obs. of 4 variables:
## $ :List of 3
## ..$ type : chr "SearchEnd"
## ..$ channel_id: int 0
## ..$ reason : chr "eof"
## $ :List of 7
## ..$ type : chr "SearchStats"
## ..$ start_time :List of 2
## ..$ update_time :List of 2
## ..$ bytes_read : int 238052
## ..$ bytes_matched : int 54486
## ..$ records_read : int 1082
## ..$ records_matched: int 384
## $ :List of 2
## ..$ type : chr "TaskEnd"
## ..$ task_id: int 0
```
## brimr Metrics
| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) |
|:-----|---------:|-----:|----:|-----:|------------:|-----:|---------:|-----:|
| R | 3 | 0.38 | 53 | 0.39 | 25 | 0.27 | 41 | 0.29 |
| Rmd | 1 | 0.12 | 15 | 0.11 | 21 | 0.23 | 30 | 0.21 |
| SUM | 4 | 0.50 | 68 | 0.50 | 46 | 0.50 | 71 | 0.50 |
clock Package Metrics for brimr
## Code of Conduct
Please note that this project is released with a Contributor Code of
Conduct. By participating in this project you agree to abide by its
terms.

16
man/brim_ast.Rd

@ -0,0 +1,16 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/brim-main.R
\name{brim_ast}
\alias{brim_ast}
\title{Turn a Brim ZQL query into an abstract syntax tree}
\usage{
brim_ast(zql, host = brim_host())
}
\arguments{
\item{zql}{the ZQL query}
\item{host}{see \code{\link[=brim_host]{brim_host()}}}
}
\description{
Turn a Brim ZQL query into an abstract syntax tree
}

13
man/brim_host.Rd

@ -0,0 +1,13 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/brim-main.R
\name{brim_host}
\alias{brim_host}
\title{Retrieve the Brim host URL}
\usage{
brim_host()
}
\description{
Looks in the \code{BRIM_HOST} environment variable for the host
URL of the Brim instance to use for the API call. This
defaults to \verb{http://127.0.0.1:9867}.
}

18
man/brim_search.Rd

@ -0,0 +1,18 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/brim-main.R
\name{brim_search}
\alias{brim_search}
\title{Post a ZQL query to the given Brim instance and retrieve processed results}
\usage{
brim_search(space_name, zql, host = brim_host())
}
\arguments{
\item{space_name}{name of the Brim space to use as the search data source}
\item{zql}{the ZQL query}
\item{host}{see \code{\link[=brim_host]{brim_host()}}}
}
\description{
Post a ZQL query to the given Brim instance and retrieve processed results
}

18
man/brim_search_raw.Rd

@ -0,0 +1,18 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/brim-main.R
\name{brim_search_raw}
\alias{brim_search_raw}
\title{Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format}
\usage{
brim_search_raw(space_name, zql, host = brim_host())
}
\arguments{
\item{space_name}{name of the Brim space to use as the search data source}
\item{zql}{the ZQL query}
\item{host}{see \code{\link[=brim_host]{brim_host()}}}
}
\description{
Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format
}

14
man/brim_spaces.Rd

@ -0,0 +1,14 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/brim-main.R
\name{brim_spaces}
\alias{brim_spaces}
\title{Retrieve active Brim spaces from the specified Brim instance}
\usage{
brim_spaces(host = brim_host())
}
\arguments{
\item{host}{see \code{\link[=brim_host]{brim_host()}}}
}
\description{
Retrieve active Brim spaces from the specified Brim instance
}

6
man/brimr.Rd

@ -4,9 +4,11 @@
\name{brimr}
\alias{brimr}
\alias{brimr-package}
\title{...}
\title{Tools to Work with Brim and zqd}
\description{
A good description goes here otherwise CRAN checks fail.
Brim (\url{https://github.com/brimsec/brim}) enables efficient query
operations on large packqet captures and log sources, such as Zeek. Tools
are provided to with with Brim components, including the Brim zqd query back-end.
}
\seealso{
Useful links:

Loading…
Cancel
Save