boB Rudis
3 years ago
15 changed files with 554 additions and 9 deletions
@ -0,0 +1,2 @@ |
|||||
|
YEAR: 2021 |
||||
|
COPYRIGHT HOLDER: Bob Rudis |
@ -0,0 +1,21 @@ |
|||||
|
# MIT License |
||||
|
|
||||
|
Copyright (c) 2021 Bob Rudis |
||||
|
|
||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
|
of this software and associated documentation files (the "Software"), to deal |
||||
|
in the Software without restriction, including without limitation the rights |
||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
|
copies of the Software, and to permit persons to whom the Software is |
||||
|
furnished to do so, subject to the following conditions: |
||||
|
|
||||
|
The above copyright notice and this permission notice shall be included in all |
||||
|
copies or substantial portions of the Software. |
||||
|
|
||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||
|
SOFTWARE. |
@ -1,4 +1,10 @@ |
|||||
# Generated by roxygen2: do not edit by hand |
# Generated by roxygen2: do not edit by hand |
||||
|
|
||||
|
export(brim_ast) |
||||
|
export(brim_host) |
||||
|
export(brim_search) |
||||
|
export(brim_search_raw) |
||||
|
export(brim_spaces) |
||||
import(httr) |
import(httr) |
||||
importFrom(jsonlite,fromJSON) |
importFrom(jsonlite,fromJSON) |
||||
|
importFrom(stringi,stri_split_lines) |
||||
|
@ -0,0 +1,100 @@ |
|||||
|
#' Retrieve the Brim host URL |
||||
|
#' |
||||
|
#' Looks in the `BRIM_HOST` environment variable for the host |
||||
|
#' URL of the Brim instance to use for the API call. This |
||||
|
#' defaults to `http://127.0.0.1:9867`. |
||||
|
#' |
||||
|
#' @export |
||||
|
brim_host <- function() { |
||||
|
Sys.getenv("BRIM_HOST", unset = "http://127.0.0.1:9867") |
||||
|
} |
||||
|
|
||||
|
#' Turn a Brim ZQL query into an abstract syntax tree |
||||
|
#' |
||||
|
#' @param zql the ZQL query |
||||
|
#' @param host see [brim_host()] |
||||
|
#' @export |
||||
|
brim_ast <- function(zql, host = brim_host()) { |
||||
|
|
||||
|
httr::POST( |
||||
|
url = sprintf("%s/ast", host), |
||||
|
httr::content_type_json(), |
||||
|
encode = "json", |
||||
|
body = list( |
||||
|
zql = zql |
||||
|
) |
||||
|
) -> res |
||||
|
|
||||
|
httr::stop_for_status(res) |
||||
|
|
||||
|
invisible(httr::content(res, "text", encoding = "UTF-8")) |
||||
|
|
||||
|
} |
||||
|
|
||||
|
#' Retrieve active Brim spaces from the specified Brim instance |
||||
|
#' |
||||
|
#' @param host see [brim_host()] |
||||
|
#' @export |
||||
|
brim_spaces <- function(host = brim_host()) { |
||||
|
|
||||
|
httr::GET( |
||||
|
url = sprintf("%s/space", host), |
||||
|
) -> res |
||||
|
|
||||
|
httr::stop_for_status(res) |
||||
|
|
||||
|
out <- httr::content(res, as = "text", encoding = "UTF-8") |
||||
|
|
||||
|
out <- jsonlite::fromJSON(out) |
||||
|
class(out) <- c("tbl_df", "tbl", "data.frame") |
||||
|
out |
||||
|
|
||||
|
} |
||||
|
|
||||
|
#' Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format |
||||
|
#' |
||||
|
#' @param space_name name of the Brim space to use as the search data source |
||||
|
#' @param zql the ZQL query |
||||
|
#' @param host see [brim_host()] |
||||
|
#' @export |
||||
|
brim_search_raw <- function(space_name, zql, host = brim_host()) { |
||||
|
|
||||
|
available_spaces <- brim_spaces() |
||||
|
space_id <- available_spaces[available_spaces[["name"]] == space_name, "id", drop=TRUE] |
||||
|
|
||||
|
x <- sprintf('{"dir":-1,"proc":%s,"space":"%s"}', trimws(brim_ast(zql)), space_id) |
||||
|
|
||||
|
httr::POST( |
||||
|
url = "http://127.0.0.1:9867/search", |
||||
|
encode = "raw", |
||||
|
httr::content_type_json(), |
||||
|
query = list( |
||||
|
format = "zjson" |
||||
|
), |
||||
|
body = x |
||||
|
) -> res |
||||
|
|
||||
|
httr::stop_for_status(res) |
||||
|
|
||||
|
out <- httr::content(res, as = "text", encoding = "UTF-8") |
||||
|
|
||||
|
out |
||||
|
|
||||
|
} |
||||
|
|
||||
|
#' Post a ZQL query to the given Brim instance and retrieve processed results |
||||
|
#' |
||||
|
#' @param space_name name of the Brim space to use as the search data source |
||||
|
#' @param zql the ZQL query |
||||
|
#' @param host see [brim_host()] |
||||
|
#' @export |
||||
|
brim_search <- function(space_name, zql, host = brim_host()) { |
||||
|
|
||||
|
res <- brim_search_raw(space_name = space_name, zql = zql, host = host) |
||||
|
res <- stringi::stri_split_lines(res, omit_empty = TRUE) |
||||
|
res <- unlist(res) |
||||
|
res <- lapply(res, jsonlite::fromJSON) |
||||
|
|
||||
|
invisible(res) |
||||
|
|
||||
|
} |
@ -1,9 +1,14 @@ |
|||||
#' ... |
#' Tools to Work with Brim and zqd |
||||
#' |
#' |
||||
|
#' Brim (<https://github.com/brimsec/brim>) enables efficient query |
||||
|
#' operations on large packqet captures and log sources, such as Zeek. Tools |
||||
|
#' are provided to with with Brim components, including the Brim zqd query back-end. |
||||
|
#' |
||||
#' @md |
#' @md |
||||
#' @name brimr |
#' @name brimr |
||||
#' @keywords internal |
#' @keywords internal |
||||
#' @author Bob Rudis (bob@@rud.is) |
#' @author Bob Rudis (bob@@rud.is) |
||||
#' @import httr |
#' @import httr |
||||
|
#' @importFrom stringi stri_split_lines |
||||
#' @importFrom jsonlite fromJSON |
#' @importFrom jsonlite fromJSON |
||||
"_PACKAGE" |
"_PACKAGE" |
||||
|
@ -0,0 +1,311 @@ |
|||||
|
|
||||
|
[![Project Status: Active – The project has reached a stable, usable |
||||
|
state and is being actively |
||||
|
developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) |
||||
|
[![Signed |
||||
|
by](https://img.shields.io/badge/Keybase-Verified-brightgreen.svg)](https://keybase.io/hrbrmstr) |
||||
|
![Signed commit |
||||
|
%](https://img.shields.io/badge/Signed_Commits-100%25-lightgrey.svg) |
||||
|
[![Linux build |
||||
|
Status](https://travis-ci.org/hrbrmstr/brimr.svg?branch=master)](https://travis-ci.org/hrbrmstr/brimr) |
||||
|
![Minimal R |
||||
|
Version](https://img.shields.io/badge/R%3E%3D-3.6.0-blue.svg) |
||||
|
![License](https://img.shields.io/badge/License-MIT-blue.svg) |
||||
|
|
||||
|
# brimr |
||||
|
|
||||
|
Tools to Work with Brim and zqd |
||||
|
|
||||
|
## Description |
||||
|
|
||||
|
Brim (<https://github.com/brimsec/brim>) enables efficient query |
||||
|
operations on large packqet captures and log sources, such as Zeek. |
||||
|
Tools are provided to with with Brim components, including the Brim zqd |
||||
|
query back-end. |
||||
|
|
||||
|
## What’s Inside The Tin |
||||
|
|
||||
|
The following functions are implemented: |
||||
|
|
||||
|
- `brim_ast`: Turn a Brim ZQL query into an abstract syntax tree |
||||
|
- `brim_host`: Retrieve the Brim host URL |
||||
|
- `brim_search_raw`: Post a ZQL query to the given Brim instance and |
||||
|
retrieve results in raq ZJSON format |
||||
|
- `brim_search`: Post a ZQL query to the given Brim instance and |
||||
|
retrieve processed results |
||||
|
- `brim_spaces`: Retrieve active Brim spaces from the specified Brim |
||||
|
instance |
||||
|
|
||||
|
## Installation |
||||
|
|
||||
|
``` r |
||||
|
remotes::install_git("https://git.rud.is/hrbrmstr/brimr.git") |
||||
|
# or |
||||
|
remotes::install_gitlab("hrbrmstr/brimr") |
||||
|
# or |
||||
|
remotes::install_bitbucket("hrbrmstr/brimr") |
||||
|
``` |
||||
|
|
||||
|
NOTE: To use the ‘remotes’ install options you will need to have the |
||||
|
[{remotes} package](https://github.com/r-lib/remotes) installed. |
||||
|
|
||||
|
## Usage |
||||
|
|
||||
|
``` r |
||||
|
library(brimr) |
||||
|
library(tibble) |
||||
|
|
||||
|
# current version |
||||
|
packageVersion("brimr") |
||||
|
## [1] '0.1.0' |
||||
|
``` |
||||
|
|
||||
|
``` r |
||||
|
brim_spaces() |
||||
|
## # A tibble: 1 x 4 |
||||
|
## id name data_path storage_kind |
||||
|
## * <chr> <chr> <chr> <chr> |
||||
|
## 1 sp_1p6pwLgtsESYBT… 2021-02-17-Trickbot-gtag-rob13-i… file:///Users/hrbrmstr/Library/Application%20Suppor… filestore |
||||
|
|
||||
|
zql <- '_path=conn | count() by id.orig_h, id.resp_h, id.resp_p | sort id.orig_h, id.resp_h, id.resp_p' |
||||
|
|
||||
|
cat(jsonlite::toJSON(jsonlite::fromJSON(brim_ast(zql)), pretty = TRUE)) |
||||
|
## { |
||||
|
## "op": ["SequentialProc"], |
||||
|
## "procs": [ |
||||
|
## { |
||||
|
## "op": "FilterProc", |
||||
|
## "filter": { |
||||
|
## "op": "CompareField", |
||||
|
## "comparator": "=", |
||||
|
## "field": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "RootRecord" |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "_path" |
||||
|
## } |
||||
|
## }, |
||||
|
## "value": { |
||||
|
## "op": "Literal", |
||||
|
## "type": "string", |
||||
|
## "value": "conn" |
||||
|
## } |
||||
|
## }, |
||||
|
## "keys": {}, |
||||
|
## "reducers": {}, |
||||
|
## "fields": {} |
||||
|
## }, |
||||
|
## { |
||||
|
## "op": "GroupByProc", |
||||
|
## "filter": { |
||||
|
## "field": { |
||||
|
## "lhs": {}, |
||||
|
## "rhs": {} |
||||
|
## }, |
||||
|
## "value": {} |
||||
|
## }, |
||||
|
## "limit": 0, |
||||
|
## "keys": [ |
||||
|
## { |
||||
|
## "op": "Assignment", |
||||
|
## "rhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "RootRecord" |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "id" |
||||
|
## } |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "orig_h" |
||||
|
## } |
||||
|
## } |
||||
|
## }, |
||||
|
## { |
||||
|
## "op": "Assignment", |
||||
|
## "rhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "RootRecord" |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "id" |
||||
|
## } |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "resp_h" |
||||
|
## } |
||||
|
## } |
||||
|
## }, |
||||
|
## { |
||||
|
## "op": "Assignment", |
||||
|
## "rhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "RootRecord" |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "id" |
||||
|
## } |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "resp_p" |
||||
|
## } |
||||
|
## } |
||||
|
## } |
||||
|
## ], |
||||
|
## "reducers": [ |
||||
|
## { |
||||
|
## "op": "Assignment", |
||||
|
## "rhs": { |
||||
|
## "op": "Reducer", |
||||
|
## "operator": "count" |
||||
|
## } |
||||
|
## } |
||||
|
## ], |
||||
|
## "fields": {} |
||||
|
## }, |
||||
|
## { |
||||
|
## "op": "SortProc", |
||||
|
## "filter": { |
||||
|
## "field": { |
||||
|
## "lhs": {}, |
||||
|
## "rhs": {} |
||||
|
## }, |
||||
|
## "value": {} |
||||
|
## }, |
||||
|
## "keys": {}, |
||||
|
## "reducers": {}, |
||||
|
## "fields": [ |
||||
|
## { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "RootRecord" |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "id" |
||||
|
## } |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "orig_h" |
||||
|
## } |
||||
|
## }, |
||||
|
## { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "RootRecord" |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "id" |
||||
|
## } |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "resp_h" |
||||
|
## } |
||||
|
## }, |
||||
|
## { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "BinaryExpr", |
||||
|
## "operator": ".", |
||||
|
## "lhs": { |
||||
|
## "op": "RootRecord" |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "id" |
||||
|
## } |
||||
|
## }, |
||||
|
## "rhs": { |
||||
|
## "op": "Identifier", |
||||
|
## "name": "resp_p" |
||||
|
## } |
||||
|
## } |
||||
|
## ], |
||||
|
## "sortdir": 1, |
||||
|
## "nullsfirst": false |
||||
|
## } |
||||
|
## ] |
||||
|
## } |
||||
|
|
||||
|
space <- "2021-02-17-Trickbot-gtag-rob13-infection-in-AD-environment.pcap" |
||||
|
|
||||
|
r <- brim_search(space, zql) |
||||
|
|
||||
|
str(r, 2) |
||||
|
## List of 5 |
||||
|
## $ :List of 2 |
||||
|
## ..$ type : chr "TaskStart" |
||||
|
## ..$ task_id: int 0 |
||||
|
## $ :List of 3 |
||||
|
## ..$ type : chr "SearchRecords" |
||||
|
## ..$ channel_id: int 0 |
||||
|
## ..$ records :'data.frame': 74 obs. of 4 variables: |
||||
|
## $ :List of 3 |
||||
|
## ..$ type : chr "SearchEnd" |
||||
|
## ..$ channel_id: int 0 |
||||
|
## ..$ reason : chr "eof" |
||||
|
## $ :List of 7 |
||||
|
## ..$ type : chr "SearchStats" |
||||
|
## ..$ start_time :List of 2 |
||||
|
## ..$ update_time :List of 2 |
||||
|
## ..$ bytes_read : int 238052 |
||||
|
## ..$ bytes_matched : int 54486 |
||||
|
## ..$ records_read : int 1082 |
||||
|
## ..$ records_matched: int 384 |
||||
|
## $ :List of 2 |
||||
|
## ..$ type : chr "TaskEnd" |
||||
|
## ..$ task_id: int 0 |
||||
|
``` |
||||
|
|
||||
|
## brimr Metrics |
||||
|
|
||||
|
| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) | |
||||
|
|:-----|---------:|-----:|----:|-----:|------------:|-----:|---------:|-----:| |
||||
|
| R | 3 | 0.38 | 53 | 0.39 | 25 | 0.27 | 41 | 0.29 | |
||||
|
| Rmd | 1 | 0.12 | 15 | 0.11 | 21 | 0.23 | 30 | 0.21 | |
||||
|
| SUM | 4 | 0.50 | 68 | 0.50 | 46 | 0.50 | 71 | 0.50 | |
||||
|
|
||||
|
clock Package Metrics for brimr |
||||
|
|
||||
|
## Code of Conduct |
||||
|
|
||||
|
Please note that this project is released with a Contributor Code of |
||||
|
Conduct. By participating in this project you agree to abide by its |
||||
|
terms. |
@ -0,0 +1,16 @@ |
|||||
|
% Generated by roxygen2: do not edit by hand |
||||
|
% Please edit documentation in R/brim-main.R |
||||
|
\name{brim_ast} |
||||
|
\alias{brim_ast} |
||||
|
\title{Turn a Brim ZQL query into an abstract syntax tree} |
||||
|
\usage{ |
||||
|
brim_ast(zql, host = brim_host()) |
||||
|
} |
||||
|
\arguments{ |
||||
|
\item{zql}{the ZQL query} |
||||
|
|
||||
|
\item{host}{see \code{\link[=brim_host]{brim_host()}}} |
||||
|
} |
||||
|
\description{ |
||||
|
Turn a Brim ZQL query into an abstract syntax tree |
||||
|
} |
@ -0,0 +1,13 @@ |
|||||
|
% Generated by roxygen2: do not edit by hand |
||||
|
% Please edit documentation in R/brim-main.R |
||||
|
\name{brim_host} |
||||
|
\alias{brim_host} |
||||
|
\title{Retrieve the Brim host URL} |
||||
|
\usage{ |
||||
|
brim_host() |
||||
|
} |
||||
|
\description{ |
||||
|
Looks in the \code{BRIM_HOST} environment variable for the host |
||||
|
URL of the Brim instance to use for the API call. This |
||||
|
defaults to \verb{http://127.0.0.1:9867}. |
||||
|
} |
@ -0,0 +1,18 @@ |
|||||
|
% Generated by roxygen2: do not edit by hand |
||||
|
% Please edit documentation in R/brim-main.R |
||||
|
\name{brim_search} |
||||
|
\alias{brim_search} |
||||
|
\title{Post a ZQL query to the given Brim instance and retrieve processed results} |
||||
|
\usage{ |
||||
|
brim_search(space_name, zql, host = brim_host()) |
||||
|
} |
||||
|
\arguments{ |
||||
|
\item{space_name}{name of the Brim space to use as the search data source} |
||||
|
|
||||
|
\item{zql}{the ZQL query} |
||||
|
|
||||
|
\item{host}{see \code{\link[=brim_host]{brim_host()}}} |
||||
|
} |
||||
|
\description{ |
||||
|
Post a ZQL query to the given Brim instance and retrieve processed results |
||||
|
} |
@ -0,0 +1,18 @@ |
|||||
|
% Generated by roxygen2: do not edit by hand |
||||
|
% Please edit documentation in R/brim-main.R |
||||
|
\name{brim_search_raw} |
||||
|
\alias{brim_search_raw} |
||||
|
\title{Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format} |
||||
|
\usage{ |
||||
|
brim_search_raw(space_name, zql, host = brim_host()) |
||||
|
} |
||||
|
\arguments{ |
||||
|
\item{space_name}{name of the Brim space to use as the search data source} |
||||
|
|
||||
|
\item{zql}{the ZQL query} |
||||
|
|
||||
|
\item{host}{see \code{\link[=brim_host]{brim_host()}}} |
||||
|
} |
||||
|
\description{ |
||||
|
Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format |
||||
|
} |
@ -0,0 +1,14 @@ |
|||||
|
% Generated by roxygen2: do not edit by hand |
||||
|
% Please edit documentation in R/brim-main.R |
||||
|
\name{brim_spaces} |
||||
|
\alias{brim_spaces} |
||||
|
\title{Retrieve active Brim spaces from the specified Brim instance} |
||||
|
\usage{ |
||||
|
brim_spaces(host = brim_host()) |
||||
|
} |
||||
|
\arguments{ |
||||
|
\item{host}{see \code{\link[=brim_host]{brim_host()}}} |
||||
|
} |
||||
|
\description{ |
||||
|
Retrieve active Brim spaces from the specified Brim instance |
||||
|
} |
Loading…
Reference in new issue