diff --git a/.Rbuildignore b/.Rbuildignore index c9a5c92..19cdbd8 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -19,3 +19,4 @@ ^CRAN-RELEASE$ ^appveyor\.yml$ ^tools$ +^LICENSE\.md$ diff --git a/DESCRIPTION b/DESCRIPTION index 2926133..8042955 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: brimr Type: Package -Title: brimr title goes here otherwise CRAN checks fail +Title: Tools to Work with Brim and zqd Version: 0.1.0 Date: 2021-02-28 Authors@R: c( @@ -8,17 +8,20 @@ Authors@R: c( comment = c(ORCID = "0000-0001-5670-2640")) ) Maintainer: Bob Rudis -Description: A good description goes here otherwise CRAN checks fail. +Description: Brim () enables efficient query + operations on large packqet captures and log sources, such as Zeek. Tools + are provided to with with Brim components, including the Brim zqd query back-end. URL: https://git.rud.is/hrbrmstr/brimr BugReports: https://git.rud.is/hrbrmstr/brimr/issues Encoding: UTF-8 -License: AGPL +License: MIT + file LICENSE Suggests: covr, tinytest Depends: - R (>= 3.5.0) + R (>= 3.6.0) Imports: httr, - jsonlite + jsonlite, + stringi Roxygen: list(markdown = TRUE) RoxygenNote: 7.1.1 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f51d1b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,2 @@ +YEAR: 2021 +COPYRIGHT HOLDER: Bob Rudis diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..c2304a0 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# MIT License + +Copyright (c) 2021 Bob Rudis + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NAMESPACE b/NAMESPACE index 5b4b9ae..b68509a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,4 +1,10 @@ # Generated by roxygen2: do not edit by hand +export(brim_ast) +export(brim_host) +export(brim_search) +export(brim_search_raw) +export(brim_spaces) import(httr) importFrom(jsonlite,fromJSON) +importFrom(stringi,stri_split_lines) diff --git a/R/brim-main.R b/R/brim-main.R new file mode 100644 index 0000000..8a3d801 --- /dev/null +++ b/R/brim-main.R @@ -0,0 +1,100 @@ +#' Retrieve the Brim host URL +#' +#' Looks in the `BRIM_HOST` environment variable for the host +#' URL of the Brim instance to use for the API call. This +#' defaults to `http://127.0.0.1:9867`. +#' +#' @export +brim_host <- function() { + Sys.getenv("BRIM_HOST", unset = "http://127.0.0.1:9867") +} + +#' Turn a Brim ZQL query into an abstract syntax tree +#' +#' @param zql the ZQL query +#' @param host see [brim_host()] +#' @export +brim_ast <- function(zql, host = brim_host()) { + + httr::POST( + url = sprintf("%s/ast", host), + httr::content_type_json(), + encode = "json", + body = list( + zql = zql + ) + ) -> res + + httr::stop_for_status(res) + + invisible(httr::content(res, "text", encoding = "UTF-8")) + +} + +#' Retrieve active Brim spaces from the specified Brim instance +#' +#' @param host see [brim_host()] +#' @export +brim_spaces <- function(host = brim_host()) { + + httr::GET( + url = sprintf("%s/space", host), + ) -> res + + httr::stop_for_status(res) + + out <- httr::content(res, as = "text", encoding = "UTF-8") + + out <- jsonlite::fromJSON(out) + class(out) <- c("tbl_df", "tbl", "data.frame") + out + +} + +#' Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format +#' +#' @param space_name name of the Brim space to use as the search data source +#' @param zql the ZQL query +#' @param host see [brim_host()] +#' @export +brim_search_raw <- function(space_name, zql, host = brim_host()) { + + available_spaces <- brim_spaces() + space_id <- available_spaces[available_spaces[["name"]] == space_name, "id", drop=TRUE] + + x <- sprintf('{"dir":-1,"proc":%s,"space":"%s"}', trimws(brim_ast(zql)), space_id) + + httr::POST( + url = "http://127.0.0.1:9867/search", + encode = "raw", + httr::content_type_json(), + query = list( + format = "zjson" + ), + body = x + ) -> res + + httr::stop_for_status(res) + + out <- httr::content(res, as = "text", encoding = "UTF-8") + + out + +} + +#' Post a ZQL query to the given Brim instance and retrieve processed results +#' +#' @param space_name name of the Brim space to use as the search data source +#' @param zql the ZQL query +#' @param host see [brim_host()] +#' @export +brim_search <- function(space_name, zql, host = brim_host()) { + + res <- brim_search_raw(space_name = space_name, zql = zql, host = host) + res <- stringi::stri_split_lines(res, omit_empty = TRUE) + res <- unlist(res) + res <- lapply(res, jsonlite::fromJSON) + + invisible(res) + +} \ No newline at end of file diff --git a/R/brimr-package.R b/R/brimr-package.R index a9d31d4..2938af9 100644 --- a/R/brimr-package.R +++ b/R/brimr-package.R @@ -1,9 +1,14 @@ -#' ... -#' +#' Tools to Work with Brim and zqd +#' +#' Brim () enables efficient query +#' operations on large packqet captures and log sources, such as Zeek. Tools +#' are provided to with with Brim components, including the Brim zqd query back-end. +#' #' @md #' @name brimr #' @keywords internal #' @author Bob Rudis (bob@@rud.is) #' @import httr +#' @importFrom stringi stri_split_lines #' @importFrom jsonlite fromJSON "_PACKAGE" diff --git a/README.Rmd b/README.Rmd index 9df440d..23fdc03 100644 --- a/README.Rmd +++ b/README.Rmd @@ -33,12 +33,27 @@ hrbrpkghelpr::install_block() ```{r lib-ex} library(brimr) +library(tibble) # current version packageVersion("brimr") ``` +```{r ex-01} +brim_spaces() + +zql <- '_path=conn | count() by id.orig_h, id.resp_h, id.resp_p | sort id.orig_h, id.resp_h, id.resp_p' + +cat(jsonlite::toJSON(jsonlite::fromJSON(brim_ast(zql)), pretty = TRUE)) + +space <- "2021-02-17-Trickbot-gtag-rob13-infection-in-AD-environment.pcap" + +r <- brim_search(space, zql) + +str(r, 2) +``` + ## brimr Metrics ```{r cloc, echo=FALSE} diff --git a/README.md b/README.md new file mode 100644 index 0000000..ca3b93c --- /dev/null +++ b/README.md @@ -0,0 +1,311 @@ + +[![Project Status: Active – The project has reached a stable, usable +state and is being actively +developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) +[![Signed +by](https://img.shields.io/badge/Keybase-Verified-brightgreen.svg)](https://keybase.io/hrbrmstr) +![Signed commit +%](https://img.shields.io/badge/Signed_Commits-100%25-lightgrey.svg) +[![Linux build +Status](https://travis-ci.org/hrbrmstr/brimr.svg?branch=master)](https://travis-ci.org/hrbrmstr/brimr) +![Minimal R +Version](https://img.shields.io/badge/R%3E%3D-3.6.0-blue.svg) +![License](https://img.shields.io/badge/License-MIT-blue.svg) + +# brimr + +Tools to Work with Brim and zqd + +## Description + +Brim () enables efficient query +operations on large packqet captures and log sources, such as Zeek. +Tools are provided to with with Brim components, including the Brim zqd +query back-end. + +## What’s Inside The Tin + +The following functions are implemented: + +- `brim_ast`: Turn a Brim ZQL query into an abstract syntax tree +- `brim_host`: Retrieve the Brim host URL +- `brim_search_raw`: Post a ZQL query to the given Brim instance and + retrieve results in raq ZJSON format +- `brim_search`: Post a ZQL query to the given Brim instance and + retrieve processed results +- `brim_spaces`: Retrieve active Brim spaces from the specified Brim + instance + +## Installation + +``` r +remotes::install_git("https://git.rud.is/hrbrmstr/brimr.git") +# or +remotes::install_gitlab("hrbrmstr/brimr") +# or +remotes::install_bitbucket("hrbrmstr/brimr") +``` + +NOTE: To use the ‘remotes’ install options you will need to have the +[{remotes} package](https://github.com/r-lib/remotes) installed. + +## Usage + +``` r +library(brimr) +library(tibble) + +# current version +packageVersion("brimr") +## [1] '0.1.0' +``` + +``` r +brim_spaces() +## # A tibble: 1 x 4 +## id name data_path storage_kind +## * +## 1 sp_1p6pwLgtsESYBT… 2021-02-17-Trickbot-gtag-rob13-i… file:///Users/hrbrmstr/Library/Application%20Suppor… filestore + +zql <- '_path=conn | count() by id.orig_h, id.resp_h, id.resp_p | sort id.orig_h, id.resp_h, id.resp_p' + +cat(jsonlite::toJSON(jsonlite::fromJSON(brim_ast(zql)), pretty = TRUE)) +## { +## "op": ["SequentialProc"], +## "procs": [ +## { +## "op": "FilterProc", +## "filter": { +## "op": "CompareField", +## "comparator": "=", +## "field": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "RootRecord" +## }, +## "rhs": { +## "op": "Identifier", +## "name": "_path" +## } +## }, +## "value": { +## "op": "Literal", +## "type": "string", +## "value": "conn" +## } +## }, +## "keys": {}, +## "reducers": {}, +## "fields": {} +## }, +## { +## "op": "GroupByProc", +## "filter": { +## "field": { +## "lhs": {}, +## "rhs": {} +## }, +## "value": {} +## }, +## "limit": 0, +## "keys": [ +## { +## "op": "Assignment", +## "rhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "RootRecord" +## }, +## "rhs": { +## "op": "Identifier", +## "name": "id" +## } +## }, +## "rhs": { +## "op": "Identifier", +## "name": "orig_h" +## } +## } +## }, +## { +## "op": "Assignment", +## "rhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "RootRecord" +## }, +## "rhs": { +## "op": "Identifier", +## "name": "id" +## } +## }, +## "rhs": { +## "op": "Identifier", +## "name": "resp_h" +## } +## } +## }, +## { +## "op": "Assignment", +## "rhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "RootRecord" +## }, +## "rhs": { +## "op": "Identifier", +## "name": "id" +## } +## }, +## "rhs": { +## "op": "Identifier", +## "name": "resp_p" +## } +## } +## } +## ], +## "reducers": [ +## { +## "op": "Assignment", +## "rhs": { +## "op": "Reducer", +## "operator": "count" +## } +## } +## ], +## "fields": {} +## }, +## { +## "op": "SortProc", +## "filter": { +## "field": { +## "lhs": {}, +## "rhs": {} +## }, +## "value": {} +## }, +## "keys": {}, +## "reducers": {}, +## "fields": [ +## { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "RootRecord" +## }, +## "rhs": { +## "op": "Identifier", +## "name": "id" +## } +## }, +## "rhs": { +## "op": "Identifier", +## "name": "orig_h" +## } +## }, +## { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "RootRecord" +## }, +## "rhs": { +## "op": "Identifier", +## "name": "id" +## } +## }, +## "rhs": { +## "op": "Identifier", +## "name": "resp_h" +## } +## }, +## { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "BinaryExpr", +## "operator": ".", +## "lhs": { +## "op": "RootRecord" +## }, +## "rhs": { +## "op": "Identifier", +## "name": "id" +## } +## }, +## "rhs": { +## "op": "Identifier", +## "name": "resp_p" +## } +## } +## ], +## "sortdir": 1, +## "nullsfirst": false +## } +## ] +## } + +space <- "2021-02-17-Trickbot-gtag-rob13-infection-in-AD-environment.pcap" + +r <- brim_search(space, zql) + +str(r, 2) +## List of 5 +## $ :List of 2 +## ..$ type : chr "TaskStart" +## ..$ task_id: int 0 +## $ :List of 3 +## ..$ type : chr "SearchRecords" +## ..$ channel_id: int 0 +## ..$ records :'data.frame': 74 obs. of 4 variables: +## $ :List of 3 +## ..$ type : chr "SearchEnd" +## ..$ channel_id: int 0 +## ..$ reason : chr "eof" +## $ :List of 7 +## ..$ type : chr "SearchStats" +## ..$ start_time :List of 2 +## ..$ update_time :List of 2 +## ..$ bytes_read : int 238052 +## ..$ bytes_matched : int 54486 +## ..$ records_read : int 1082 +## ..$ records_matched: int 384 +## $ :List of 2 +## ..$ type : chr "TaskEnd" +## ..$ task_id: int 0 +``` + +## brimr Metrics + +| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) | +|:-----|---------:|-----:|----:|-----:|------------:|-----:|---------:|-----:| +| R | 3 | 0.38 | 53 | 0.39 | 25 | 0.27 | 41 | 0.29 | +| Rmd | 1 | 0.12 | 15 | 0.11 | 21 | 0.23 | 30 | 0.21 | +| SUM | 4 | 0.50 | 68 | 0.50 | 46 | 0.50 | 71 | 0.50 | + +clock Package Metrics for brimr + +## Code of Conduct + +Please note that this project is released with a Contributor Code of +Conduct. By participating in this project you agree to abide by its +terms. diff --git a/man/brim_ast.Rd b/man/brim_ast.Rd new file mode 100644 index 0000000..357bd93 --- /dev/null +++ b/man/brim_ast.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/brim-main.R +\name{brim_ast} +\alias{brim_ast} +\title{Turn a Brim ZQL query into an abstract syntax tree} +\usage{ +brim_ast(zql, host = brim_host()) +} +\arguments{ +\item{zql}{the ZQL query} + +\item{host}{see \code{\link[=brim_host]{brim_host()}}} +} +\description{ +Turn a Brim ZQL query into an abstract syntax tree +} diff --git a/man/brim_host.Rd b/man/brim_host.Rd new file mode 100644 index 0000000..4cff1ed --- /dev/null +++ b/man/brim_host.Rd @@ -0,0 +1,13 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/brim-main.R +\name{brim_host} +\alias{brim_host} +\title{Retrieve the Brim host URL} +\usage{ +brim_host() +} +\description{ +Looks in the \code{BRIM_HOST} environment variable for the host +URL of the Brim instance to use for the API call. This +defaults to \verb{http://127.0.0.1:9867}. +} diff --git a/man/brim_search.Rd b/man/brim_search.Rd new file mode 100644 index 0000000..bfceb86 --- /dev/null +++ b/man/brim_search.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/brim-main.R +\name{brim_search} +\alias{brim_search} +\title{Post a ZQL query to the given Brim instance and retrieve processed results} +\usage{ +brim_search(space_name, zql, host = brim_host()) +} +\arguments{ +\item{space_name}{name of the Brim space to use as the search data source} + +\item{zql}{the ZQL query} + +\item{host}{see \code{\link[=brim_host]{brim_host()}}} +} +\description{ +Post a ZQL query to the given Brim instance and retrieve processed results +} diff --git a/man/brim_search_raw.Rd b/man/brim_search_raw.Rd new file mode 100644 index 0000000..7b3236e --- /dev/null +++ b/man/brim_search_raw.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/brim-main.R +\name{brim_search_raw} +\alias{brim_search_raw} +\title{Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format} +\usage{ +brim_search_raw(space_name, zql, host = brim_host()) +} +\arguments{ +\item{space_name}{name of the Brim space to use as the search data source} + +\item{zql}{the ZQL query} + +\item{host}{see \code{\link[=brim_host]{brim_host()}}} +} +\description{ +Post a ZQL query to the given Brim instance and retrieve results in raq ZJSON format +} diff --git a/man/brim_spaces.Rd b/man/brim_spaces.Rd new file mode 100644 index 0000000..3e2e3d8 --- /dev/null +++ b/man/brim_spaces.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/brim-main.R +\name{brim_spaces} +\alias{brim_spaces} +\title{Retrieve active Brim spaces from the specified Brim instance} +\usage{ +brim_spaces(host = brim_host()) +} +\arguments{ +\item{host}{see \code{\link[=brim_host]{brim_host()}}} +} +\description{ +Retrieve active Brim spaces from the specified Brim instance +} diff --git a/man/brimr.Rd b/man/brimr.Rd index 6bedd39..80bc796 100644 --- a/man/brimr.Rd +++ b/man/brimr.Rd @@ -4,9 +4,11 @@ \name{brimr} \alias{brimr} \alias{brimr-package} -\title{...} +\title{Tools to Work with Brim and zqd} \description{ -A good description goes here otherwise CRAN checks fail. +Brim (\url{https://github.com/brimsec/brim}) enables efficient query +operations on large packqet captures and log sources, such as Zeek. Tools +are provided to with with Brim components, including the Brim zqd query back-end. } \seealso{ Useful links: