Browse Source

zq command line utility function

master
boB Rudis 3 years ago
parent
commit
f8f02391b9
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
  1. 5
      DESCRIPTION
  2. 2
      NAMESPACE
  3. 1
      R/brimr-package.R
  4. 49
      R/utils.R
  5. 63
      R/zq-cmd.R
  6. 11
      README.Rmd
  7. 34
      README.md
  8. BIN
      inst/logs/conn.log.gz
  9. 44
      man/zq_cmd.Rd

5
DESCRIPTION

@ -16,7 +16,7 @@ BugReports: https://git.rud.is/hrbrmstr/brimr/issues
Encoding: UTF-8
License: MIT + file LICENSE
Suggests:
covr, tinytest
covr, tinytest, data.tab.e
Depends:
R (>= 3.6.0)
Imports:
@ -24,6 +24,7 @@ Imports:
jsonlite,
scales,
ipaddress,
stringi
stringi,
ndjson
Roxygen: list(markdown = TRUE)
RoxygenNote: 7.1.1

2
NAMESPACE

@ -7,8 +7,10 @@ export(brim_search)
export(brim_search_raw)
export(brim_spaces)
export(tidy_brim)
export(zq_cmd)
import(httr)
import(ipaddress)
importFrom(jsonlite,fromJSON)
importFrom(ndjson,stream_in)
importFrom(scales,comma)
importFrom(stringi,stri_split_lines)

1
R/brimr-package.R

@ -9,6 +9,7 @@
#' @keywords internal
#' @author Bob Rudis (bob@@rud.is)
#' @import httr ipaddress
#' @importFrom ndjson stream_in
#' @importFrom scales comma
#' @importFrom stringi stri_split_lines
#' @importFrom jsonlite fromJSON

49
R/utils.R

@ -1,4 +1,53 @@
is_windows <- function() .Platform$OS.type == "windows"
is_mac <- function() Sys.info()[["sysname"]] == "Darwin"
is_linux <- function() Sys.info()[["sysname"]] == "Linux"
platform <- function() {
if (is_windows()) return("win")
if (is_mac()) return("mac")
if (is_linux()) return("linux")
stop("unknown platform")
}
set_names <- function (object = nm, nm) {
names(object) <- nm
object
}
version_test <- function(zq_path) {
try(
system2(
command = zq_path,
args = "-version",
stdout = TRUE,
stderr = TRUE
) ,
silent = TRUE
) -> res
((is.null(attributes(res))) && any(grepl("Version", res)))
}
find_zq <- function() {
try_env <- Sys.getenv("ZQ_PATH", unset = NA)
if (is.na(try_env)) {
if (is_mac()) {
try_env <- "/Applications/Brim.app/Contents/Resources/app/zdeps/zq"
} else if (is_linux()) {
try_env <- "/usr/lib/brim/resources/app/zdeps/zq"
}
} # TODO Windows
if (!file.exists(try_env)) try_env <- Sys.which("zq")
if (file.exists(try_env)) {
if (version_test(try_env)) return(try_env)
}
NA
}

63
R/zq-cmd.R

@ -0,0 +1,63 @@
#' Execute a zq command line
#'
#' zq is a command-line tool for processing logs. It applies boolean logic to
#' filter each log value, optionally computes analytics and transformations,
#' and returns results that can be consumed programmatically.\cr
#' \cr
#' This function takes command line arguments to be used with `zq` (
#' in the form [system2()] uses) and reads the results into a
#' data frame (it is actually a `data.table`).\cr
#' \cr
#' If the environment variable `ZQ_PATH` is not set or invalid, this function
#' will attempt to guess the `zq` binary path.\cr
#' \cr
#' Do not specify an output format as `-f ndjson` is added by default.
#'
#' @param args see [system2()]
#' @param parse if `TRUE` (the default) the output of the command line call
#' will be parsed using [ndjson::stream_in()]. There are some combinations
#' of `zq` flags that will never return ndjson output. There are heuristics
#' in place to detect this, but you can deliberately force the function
#' to return raw command line output by setting this to `FALSE`.
#' @return `data.table` (if output is parseable); character vector (if output is
#' either not parseable or `parse` equals `FALSE`); or `NULL` in the
#' event an error occurred when processing the `zq` command line.
#' @export
#' @examples
#' zq_cmd(
#' c(
#' '"* | cut ts,id.orig_h,id.orig_p"', # note the quotes
#' system.file("logs", "conn.log.gz", package = "brimr")
#' )
#' )
zq_cmd <- function(args, parse = TRUE) {
zq_path <- find_zq()
if (is.na(zq_path)) {
stop(
"Cannot locate 'zq'. Please set the ZQ_PATH to the full path to the ",
"executable or install Brim Desktop and/or zq.", call. = FALSE
)
}
tf <- tempfile(fileext = ".json")
on.exit(unlink(tf))
system2(
command = zq_path,
args = c("-f ", "ndjson", args),
stdout = tf
) -> res
if (!is.null(res)) {
one <- readLines(tf, 1)
if ((parse) && grepl("\\{", one)) return(ndjson::stream_in(tf))
readLines(tf)
}
}

11
README.Rmd

@ -114,6 +114,17 @@ ggraph(g, layout = "linear") +
)
```
### Use `zq` directly
```{r zq_cmdline, cache=TRUE}
zq_cmd(
c(
'"* | cut ts,id.orig_h,id.orig_p"', # note the quotes
system.file("logs", "conn.log.gz", package = "brimr")
)
)
```
## brimr Metrics
```{r cloc, echo=FALSE}

34
README.md

@ -36,6 +36,7 @@ The following functions are implemented:
- `brim_spaces`: Retrieve active Brim spaces from the specified Brim
instance
- `tidy_brim`: Turn Brim/zqd search results into a data frame
- `zq_cmd`: Execute a zq command line
## Installation
@ -180,13 +181,36 @@ ggraph(g, layout = "linear") +
<img src="man/figures/README-graph-1.png" width="864" />
### Use `zq` directly
``` r
zq_cmd(
c(
'"* | cut ts,id.orig_h,id.orig_p"', # note the quotes
system.file("logs", "conn.log.gz", package = "brimr")
)
)
## id.orig_h id.orig_p ts
## 1: 10.164.94.120 39681 2018-03-24T17:15:21.255387Z
## 2: 10.47.25.80 50817 2018-03-24T17:15:21.411148Z
## 3: 10.47.25.80 50817 2018-03-24T17:15:21.926018Z
## 4: 10.47.25.80 50813 2018-03-24T17:15:22.690601Z
## 5: 10.47.25.80 50813 2018-03-24T17:15:23.205187Z
## ---
## 988: 10.174.251.215 33003 2018-03-24T17:15:21.429238Z
## 989: 10.174.251.215 33003 2018-03-24T17:15:21.429315Z
## 990: 10.174.251.215 33003 2018-03-24T17:15:21.429479Z
## 991: 10.164.94.120 38265 2018-03-24T17:15:21.427375Z
## 992: 10.174.251.215 33003 2018-03-24T17:15:21.433306Z
```
## brimr Metrics
| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) |
|:-----|---------:|----:|----:|-----:|------------:|-----:|---------:|-----:|
| R | 4 | 0.4 | 123 | 0.36 | 48 | 0.29 | 53 | 0.27 |
| Rmd | 1 | 0.1 | 47 | 0.14 | 35 | 0.21 | 44 | 0.23 |
| SUM | 5 | 0.5 | 170 | 0.50 | 83 | 0.50 | 97 | 0.50 |
| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) |
|:-----|---------:|-----:|----:|-----:|------------:|-----:|---------:|-----:|
| R | 5 | 0.42 | 180 | 0.39 | 71 | 0.33 | 86 | 0.32 |
| Rmd | 1 | 0.08 | 53 | 0.11 | 37 | 0.17 | 47 | 0.18 |
| SUM | 6 | 0.50 | 233 | 0.50 | 108 | 0.50 | 133 | 0.50 |
clock Package Metrics for brimr

BIN
inst/logs/conn.log.gz

Binary file not shown.

44
man/zq_cmd.Rd

@ -0,0 +1,44 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/zq-cmd.R
\name{zq_cmd}
\alias{zq_cmd}
\title{Execute a zq command line}
\usage{
zq_cmd(args, parse = TRUE)
}
\arguments{
\item{args}{see \code{\link[=system2]{system2()}}}
\item{parse}{if \code{TRUE} (the default) the output of the command line call
will be parsed using \code{\link[ndjson:stream_in]{ndjson::stream_in()}}. There are some combinations
of \code{zq} flags that will never return ndjson output. There are heuristics
in place to detect this, but you can deliberately force the function
to return raw command line output by setting this to \code{FALSE}.}
}
\value{
\code{data.table} (if output is parseable); character vector (if output is
either not parseable or \code{parse} equals \code{FALSE}); or \code{NULL} in the
event an error occurred when processing the \code{zq} command line.
}
\description{
zq is a command-line tool for processing logs. It applies boolean logic to
filter each log value, optionally computes analytics and transformations,
and returns results that can be consumed programmatically.\cr
\cr
This function takes command line arguments to be used with \code{zq} (
in the form \code{\link[=system2]{system2()}} uses) and reads the results into a
data frame (it is actually a \code{data.table}).\cr
\cr
If the environment variable \code{ZQ_PATH} is not set or invalid, this function
will attempt to guess the \code{zq} binary path.\cr
\cr
Do not specify an output format as \verb{-f ndjson} is added by default.
}
\examples{
zq_cmd(
c(
'"* | cut ts,id.orig_h,id.orig_p"', # note the quotes
system.file("logs", "conn.log.gz", package = "brimr")
)
)
}
Loading…
Cancel
Save