15 changed files with 554 additions and 9 deletions
@ -0,0 +1,2 @@ |
|||
YEAR: 2021 |
|||
COPYRIGHT HOLDER: Bob Rudis |
|||
@ -0,0 +1,21 @@ |
|||
# MIT License |
|||
|
|||
Copyright (c) 2021 Bob Rudis |
|||
|
|||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|||
of this software and associated documentation files (the "Software"), to deal |
|||
in the Software without restriction, including without limitation the rights |
|||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|||
copies of the Software, and to permit persons to whom the Software is |
|||
furnished to do so, subject to the following conditions: |
|||
|
|||
The above copyright notice and this permission notice shall be included in all |
|||
copies or substantial portions of the Software. |
|||
|
|||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
|||
SOFTWARE. |
|||
@ -1,4 +1,10 @@ |
|||
# Generated by roxygen2: do not edit by hand |
|||
|
|||
export(brim_ast) |
|||
export(brim_host) |
|||
export(brim_search) |
|||
export(brim_search_raw) |
|||
export(brim_spaces) |
|||
import(httr) |
|||
importFrom(jsonlite,fromJSON) |
|||
importFrom(stringi,stri_split_lines) |
|||
|
|||
@ -0,0 +1,100 @@ |
|||
#' Retrieve the Brim host URL |
|||
#' |
|||
#' Looks in the `BRIM_HOST` environment variable for the host |
|||
#' URL of the Brim instance to use for the API call. This |
|||
#' defaults to `http://127.0.0.1:9867`. |
|||
#' |
|||
#' @export |
|||
brim_host <- function() { |
|||
Sys.getenv("BRIM_HOST", unset = "http://127.0.0.1:9867") |
|||
} |
|||
|
|||
#' Turn a Brim ZQL query into an abstract syntax tree |
|||
#' |
|||
#' @param zql the ZQL query |
|||
#' @param host see [brim_host()] |
|||
#' @export |
|||
brim_ast <- function(zql, host = brim_host()) { |
|||
|
|||
httr::POST( |
|||
url = sprintf("%s/ast", host), |
|||
httr::content_type_json(), |
|||
encode = "json", |
|||
body = list( |
|||
zql = zql |
|||
) |
|||
) -> res |
|||
|
|||
httr::stop_for_status(res) |
|||
|
|||
invisible(httr::content(res, "text", encoding = "UTF-8")) |
|||
|
|||
} |
|||
|
|||
#' Retrieve active Brim spaces from the specified Brim instance |
|||
#' |
|||
#' @param host see [brim_host()] |
|||
#' @export |
|||
brim_spaces <- function(host = brim_host()) { |
|||
|
|||
httr::GET( |
|||
url = sprintf("%s/space", host), |
|||
) -> res |
|||
|
|||
httr::stop_for_status(res) |
|||
|
|||
out <- httr::content(res, as = "text", encoding = "UTF-8") |
|||
|
|||
out <- jsonlite::fromJSON(out) |
|||
class(out) <- c("tbl_df", "tbl", "data.frame") |
|||
out |
|||
|
|||
} |
|||
|
|||
#' Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format |
|||
#' |
|||
#' @param space_name name of the Brim space to use as the search data source |
|||
#' @param zql the ZQL query |
|||
#' @param host see [brim_host()] |
|||
#' @export |
|||
brim_search_raw <- function(space_name, zql, host = brim_host()) { |
|||
|
|||
available_spaces <- brim_spaces() |
|||
space_id <- available_spaces[available_spaces[["name"]] == space_name, "id", drop=TRUE] |
|||
|
|||
x <- sprintf('{"dir":-1,"proc":%s,"space":"%s"}', trimws(brim_ast(zql)), space_id) |
|||
|
|||
httr::POST( |
|||
url = "http://127.0.0.1:9867/search", |
|||
encode = "raw", |
|||
httr::content_type_json(), |
|||
query = list( |
|||
format = "zjson" |
|||
), |
|||
body = x |
|||
) -> res |
|||
|
|||
httr::stop_for_status(res) |
|||
|
|||
out <- httr::content(res, as = "text", encoding = "UTF-8") |
|||
|
|||
out |
|||
|
|||
} |
|||
|
|||
#' Post a ZQL query to the given Brim instance and retrieve processed results |
|||
#' |
|||
#' @param space_name name of the Brim space to use as the search data source |
|||
#' @param zql the ZQL query |
|||
#' @param host see [brim_host()] |
|||
#' @export |
|||
brim_search <- function(space_name, zql, host = brim_host()) { |
|||
|
|||
res <- brim_search_raw(space_name = space_name, zql = zql, host = host) |
|||
res <- stringi::stri_split_lines(res, omit_empty = TRUE) |
|||
res <- unlist(res) |
|||
res <- lapply(res, jsonlite::fromJSON) |
|||
|
|||
invisible(res) |
|||
|
|||
} |
|||
@ -1,9 +1,14 @@ |
|||
#' ... |
|||
#' |
|||
#' Tools to Work with Brim and zqd |
|||
#' |
|||
#' Brim (<https://github.com/brimsec/brim>) enables efficient query |
|||
#' operations on large packqet captures and log sources, such as Zeek. Tools |
|||
#' are provided to with with Brim components, including the Brim zqd query back-end. |
|||
#' |
|||
#' @md |
|||
#' @name brimr |
|||
#' @keywords internal |
|||
#' @author Bob Rudis (bob@@rud.is) |
|||
#' @import httr |
|||
#' @importFrom stringi stri_split_lines |
|||
#' @importFrom jsonlite fromJSON |
|||
"_PACKAGE" |
|||
|
|||
@ -0,0 +1,311 @@ |
|||
|
|||
[](https://www.repostatus.org/#active) |
|||
[](https://keybase.io/hrbrmstr) |
|||
 |
|||
[](https://travis-ci.org/hrbrmstr/brimr) |
|||
 |
|||
 |
|||
|
|||
# brimr |
|||
|
|||
Tools to Work with Brim and zqd |
|||
|
|||
## Description |
|||
|
|||
Brim (<https://github.com/brimsec/brim>) enables efficient query |
|||
operations on large packqet captures and log sources, such as Zeek. |
|||
Tools are provided to with with Brim components, including the Brim zqd |
|||
query back-end. |
|||
|
|||
## What’s Inside The Tin |
|||
|
|||
The following functions are implemented: |
|||
|
|||
- `brim_ast`: Turn a Brim ZQL query into an abstract syntax tree |
|||
- `brim_host`: Retrieve the Brim host URL |
|||
- `brim_search_raw`: Post a ZQL query to the given Brim instance and |
|||
retrieve results in raq ZJSON format |
|||
- `brim_search`: Post a ZQL query to the given Brim instance and |
|||
retrieve processed results |
|||
- `brim_spaces`: Retrieve active Brim spaces from the specified Brim |
|||
instance |
|||
|
|||
## Installation |
|||
|
|||
``` r |
|||
remotes::install_git("https://git.rud.is/hrbrmstr/brimr.git") |
|||
# or |
|||
remotes::install_gitlab("hrbrmstr/brimr") |
|||
# or |
|||
remotes::install_bitbucket("hrbrmstr/brimr") |
|||
``` |
|||
|
|||
NOTE: To use the ‘remotes’ install options you will need to have the |
|||
[{remotes} package](https://github.com/r-lib/remotes) installed. |
|||
|
|||
## Usage |
|||
|
|||
``` r |
|||
library(brimr) |
|||
library(tibble) |
|||
|
|||
# current version |
|||
packageVersion("brimr") |
|||
## [1] '0.1.0' |
|||
``` |
|||
|
|||
``` r |
|||
brim_spaces() |
|||
## # A tibble: 1 x 4 |
|||
## id name data_path storage_kind |
|||
## * <chr> <chr> <chr> <chr> |
|||
## 1 sp_1p6pwLgtsESYBT… 2021-02-17-Trickbot-gtag-rob13-i… file:///Users/hrbrmstr/Library/Application%20Suppor… filestore |
|||
|
|||
zql <- '_path=conn | count() by id.orig_h, id.resp_h, id.resp_p | sort id.orig_h, id.resp_h, id.resp_p' |
|||
|
|||
cat(jsonlite::toJSON(jsonlite::fromJSON(brim_ast(zql)), pretty = TRUE)) |
|||
## { |
|||
## "op": ["SequentialProc"], |
|||
## "procs": [ |
|||
## { |
|||
## "op": "FilterProc", |
|||
## "filter": { |
|||
## "op": "CompareField", |
|||
## "comparator": "=", |
|||
## "field": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "RootRecord" |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "_path" |
|||
## } |
|||
## }, |
|||
## "value": { |
|||
## "op": "Literal", |
|||
## "type": "string", |
|||
## "value": "conn" |
|||
## } |
|||
## }, |
|||
## "keys": {}, |
|||
## "reducers": {}, |
|||
## "fields": {} |
|||
## }, |
|||
## { |
|||
## "op": "GroupByProc", |
|||
## "filter": { |
|||
## "field": { |
|||
## "lhs": {}, |
|||
## "rhs": {} |
|||
## }, |
|||
## "value": {} |
|||
## }, |
|||
## "limit": 0, |
|||
## "keys": [ |
|||
## { |
|||
## "op": "Assignment", |
|||
## "rhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "RootRecord" |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "id" |
|||
## } |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "orig_h" |
|||
## } |
|||
## } |
|||
## }, |
|||
## { |
|||
## "op": "Assignment", |
|||
## "rhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "RootRecord" |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "id" |
|||
## } |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "resp_h" |
|||
## } |
|||
## } |
|||
## }, |
|||
## { |
|||
## "op": "Assignment", |
|||
## "rhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "RootRecord" |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "id" |
|||
## } |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "resp_p" |
|||
## } |
|||
## } |
|||
## } |
|||
## ], |
|||
## "reducers": [ |
|||
## { |
|||
## "op": "Assignment", |
|||
## "rhs": { |
|||
## "op": "Reducer", |
|||
## "operator": "count" |
|||
## } |
|||
## } |
|||
## ], |
|||
## "fields": {} |
|||
## }, |
|||
## { |
|||
## "op": "SortProc", |
|||
## "filter": { |
|||
## "field": { |
|||
## "lhs": {}, |
|||
## "rhs": {} |
|||
## }, |
|||
## "value": {} |
|||
## }, |
|||
## "keys": {}, |
|||
## "reducers": {}, |
|||
## "fields": [ |
|||
## { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "RootRecord" |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "id" |
|||
## } |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "orig_h" |
|||
## } |
|||
## }, |
|||
## { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "RootRecord" |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "id" |
|||
## } |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "resp_h" |
|||
## } |
|||
## }, |
|||
## { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "BinaryExpr", |
|||
## "operator": ".", |
|||
## "lhs": { |
|||
## "op": "RootRecord" |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "id" |
|||
## } |
|||
## }, |
|||
## "rhs": { |
|||
## "op": "Identifier", |
|||
## "name": "resp_p" |
|||
## } |
|||
## } |
|||
## ], |
|||
## "sortdir": 1, |
|||
## "nullsfirst": false |
|||
## } |
|||
## ] |
|||
## } |
|||
|
|||
space <- "2021-02-17-Trickbot-gtag-rob13-infection-in-AD-environment.pcap" |
|||
|
|||
r <- brim_search(space, zql) |
|||
|
|||
str(r, 2) |
|||
## List of 5 |
|||
## $ :List of 2 |
|||
## ..$ type : chr "TaskStart" |
|||
## ..$ task_id: int 0 |
|||
## $ :List of 3 |
|||
## ..$ type : chr "SearchRecords" |
|||
## ..$ channel_id: int 0 |
|||
## ..$ records :'data.frame': 74 obs. of 4 variables: |
|||
## $ :List of 3 |
|||
## ..$ type : chr "SearchEnd" |
|||
## ..$ channel_id: int 0 |
|||
## ..$ reason : chr "eof" |
|||
## $ :List of 7 |
|||
## ..$ type : chr "SearchStats" |
|||
## ..$ start_time :List of 2 |
|||
## ..$ update_time :List of 2 |
|||
## ..$ bytes_read : int 238052 |
|||
## ..$ bytes_matched : int 54486 |
|||
## ..$ records_read : int 1082 |
|||
## ..$ records_matched: int 384 |
|||
## $ :List of 2 |
|||
## ..$ type : chr "TaskEnd" |
|||
## ..$ task_id: int 0 |
|||
``` |
|||
|
|||
## brimr Metrics |
|||
|
|||
| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) | |
|||
|:-----|---------:|-----:|----:|-----:|------------:|-----:|---------:|-----:| |
|||
| R | 3 | 0.38 | 53 | 0.39 | 25 | 0.27 | 41 | 0.29 | |
|||
| Rmd | 1 | 0.12 | 15 | 0.11 | 21 | 0.23 | 30 | 0.21 | |
|||
| SUM | 4 | 0.50 | 68 | 0.50 | 46 | 0.50 | 71 | 0.50 | |
|||
|
|||
clock Package Metrics for brimr |
|||
|
|||
## Code of Conduct |
|||
|
|||
Please note that this project is released with a Contributor Code of |
|||
Conduct. By participating in this project you agree to abide by its |
|||
terms. |
|||
@ -0,0 +1,16 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/brim-main.R |
|||
\name{brim_ast} |
|||
\alias{brim_ast} |
|||
\title{Turn a Brim ZQL query into an abstract syntax tree} |
|||
\usage{ |
|||
brim_ast(zql, host = brim_host()) |
|||
} |
|||
\arguments{ |
|||
\item{zql}{the ZQL query} |
|||
|
|||
\item{host}{see \code{\link[=brim_host]{brim_host()}}} |
|||
} |
|||
\description{ |
|||
Turn a Brim ZQL query into an abstract syntax tree |
|||
} |
|||
@ -0,0 +1,13 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/brim-main.R |
|||
\name{brim_host} |
|||
\alias{brim_host} |
|||
\title{Retrieve the Brim host URL} |
|||
\usage{ |
|||
brim_host() |
|||
} |
|||
\description{ |
|||
Looks in the \code{BRIM_HOST} environment variable for the host |
|||
URL of the Brim instance to use for the API call. This |
|||
defaults to \verb{http://127.0.0.1:9867}. |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/brim-main.R |
|||
\name{brim_search} |
|||
\alias{brim_search} |
|||
\title{Post a ZQL query to the given Brim instance and retrieve processed results} |
|||
\usage{ |
|||
brim_search(space_name, zql, host = brim_host()) |
|||
} |
|||
\arguments{ |
|||
\item{space_name}{name of the Brim space to use as the search data source} |
|||
|
|||
\item{zql}{the ZQL query} |
|||
|
|||
\item{host}{see \code{\link[=brim_host]{brim_host()}}} |
|||
} |
|||
\description{ |
|||
Post a ZQL query to the given Brim instance and retrieve processed results |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/brim-main.R |
|||
\name{brim_search_raw} |
|||
\alias{brim_search_raw} |
|||
\title{Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format} |
|||
\usage{ |
|||
brim_search_raw(space_name, zql, host = brim_host()) |
|||
} |
|||
\arguments{ |
|||
\item{space_name}{name of the Brim space to use as the search data source} |
|||
|
|||
\item{zql}{the ZQL query} |
|||
|
|||
\item{host}{see \code{\link[=brim_host]{brim_host()}}} |
|||
} |
|||
\description{ |
|||
Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format |
|||
} |
|||
@ -0,0 +1,14 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/brim-main.R |
|||
\name{brim_spaces} |
|||
\alias{brim_spaces} |
|||
\title{Retrieve active Brim spaces from the specified Brim instance} |
|||
\usage{ |
|||
brim_spaces(host = brim_host()) |
|||
} |
|||
\arguments{ |
|||
\item{host}{see \code{\link[=brim_host]{brim_host()}}} |
|||
} |
|||
\description{ |
|||
Retrieve active Brim spaces from the specified Brim instance |
|||
} |
|||
Loading…
Reference in new issue