diff --git a/DESCRIPTION b/DESCRIPTION index 1532257..9a4ef95 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -5,13 +5,14 @@ Version: 0.2.0 Date: 2018-03-19 Authors@R: c( person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"), - comment = c(ORCID = "0000-0001-5670-2640")), - person("Zachary, "Kurtz", email="zdkurtz@gmail.com", role = "ctb") + comment = c(ORCID = "0000-0001-5670-2640")), + person("Zachary", "Kurtz", email = "zdkurtz@gmail.com", role = "ctb") ) Maintainer: Bob Rudis Encoding: UTF-8 -Description: Helpers for Accessing and Querying Amazon Athena. Including a - lightweight RJDBC shim. +Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, + perform queries and retrieve query results. A lightweight 'RJDBC' implementation + is included along with an interface to the 'AWS' command-line utility. URL: https://github.com/hrbrmstr/metis BugReports: https://github.com/hrbrmstr/metis/issues License: AGPL @@ -26,5 +27,8 @@ Imports: DBI, dplyr, readr, - aws.signature + aws.signature, + uuid, + sys, + jsonlite RoxygenNote: 6.0.1.9000 diff --git a/NAMESPACE b/NAMESPACE index bfda388..e743d02 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -19,5 +19,8 @@ import(RJDBC) import(dplyr) importFrom(aws.signature,read_credentials) importFrom(aws.signature,use_credentials) +importFrom(jsonlite,fromJSON) importFrom(rJava,.jcall) importFrom(readr,type_convert) +importFrom(sys,exec_internal) +importFrom(uuid,UUIDgenerate) diff --git a/R/awscli-util.R b/R/awscli-util.R new file mode 100644 index 0000000..905fd63 --- /dev/null +++ b/R/awscli-util.R @@ -0,0 +1,26 @@ +.aws_bin <- function() { + unname(Sys.which('aws')) +} + +.athenacli <- function(...) { + + args <- c("athena") + + in_args <- list(...) + if (length(in_args) == 0) in_args <- "help" + + args <- c(args, unlist(in_args, use.names=FALSE)) + + res <- sys::exec_internal(.aws_bin(), args = args, error = FALSE) + + if (length(res$stdout) > 0) { + + out <- rawToChar(res$stdout) + + if ("help" %in% args) cat(out, sep="") + + invisible(out) + + } + +} diff --git a/R/jdbc.r b/R/jdbc.r index 6292535..591f869 100644 --- a/R/jdbc.r +++ b/R/jdbc.r @@ -4,18 +4,24 @@ setClass( "AthenaDriver", - representation("JDBCDriver", identifier.quote="character", jdrv="jobjRef") + representation( + "JDBCDriver", + identifier.quote = "character", + jdrv = "jobjRef" + ) ) #' AthenaJDBC #' #' @export -Athena <- function(identifier.quote='`') { +Athena <- function(identifier.quote = '`') { - drv <- JDBC(driverClass="com.amazonaws.athena.jdbc.AthenaDriver", - system.file("AthenaJDBC41-1.1.0.jar", package="metis"), - identifier.quote=identifier.quote) + JDBC( + driverClass = "com.amazonaws.athena.jdbc.AthenaDriver", + system.file("java", "AthenaJDBC41-1.1.0.jar", package = "metis"), + identifier.quote = identifier.quote + ) -> drv return(as(drv, "AthenaDriver")) @@ -57,18 +63,21 @@ setMethod( 'jdbc:awsathena://athena.%s.amazonaws.com:443/%s', region, schema_name ) - jc <- callNextMethod(drv, conn_string, - s3_staging_dir = s3_staging_dir, - schema_name = schema_name, - max_error_retries = max_error_retries, - connection_timeout = connection_timeout, - socket_timeout = socket_timeout, - retry_base_delay = retry_base_delay, - retry_max_backoff_time = retry_max_backoff_time, - log_path = log_path, - log_level = log_level, - aws_credentials_provider_class = provider, - ...) + callNextMethod( + drv, + conn_string, + s3_staging_dir = s3_staging_dir, + schema_name = schema_name, + max_error_retries = max_error_retries, + connection_timeout = connection_timeout, + socket_timeout = socket_timeout, + retry_base_delay = retry_base_delay, + retry_max_backoff_time = retry_max_backoff_time, + log_path = log_path, + log_level = log_level, + aws_credentials_provider_class = provider, + ... + ) -> jc return(as(jc, "AthenaConnection")) @@ -97,7 +106,7 @@ setMethod( "dbSendQuery", signature(conn="AthenaConnection", statement="character"), - def = function(conn, statement, ...) { + definition = function(conn, statement, ...) { return(as(callNextMethod(), "AthenaResult")) } @@ -115,7 +124,7 @@ setMethod( "dbGetQuery", signature(conn="AthenaConnection", statement="character"), - def = function(conn, statement, type_convert=FALSE, ...) { + definition = function(conn, statement, type_convert=FALSE, ...) { r <- dbSendQuery(conn, statement, ...) on.exit(.jcall(r@stat, "V", "close")) res <- dplyr::tbl_df(fetch(r, -1, block=1000)) @@ -136,7 +145,7 @@ setMethod( "dbListTables", signature(conn="AthenaConnection"), - def = function(conn, pattern='*', schema, ...) { + definition = function(conn, pattern='*', schema, ...) { if (missing(pattern)) { dbGetQuery( @@ -165,7 +174,7 @@ setMethod( "dbExistsTable", signature(conn="AthenaConnection", name="character"), - def = function(conn, name, schema, ...) { + definition = function(conn, name, schema, ...) { length(dbListTables(conn, schema=schema, pattern=name)) > 0 } @@ -183,7 +192,7 @@ setMethod( "dbListFields", signature(conn="AthenaConnection", name="character"), - def = function(conn, name, schema, ...) { + definition = function(conn, name, schema, ...) { query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, name) res <- dbGetQuery(conn, query) colnames(res) @@ -203,7 +212,7 @@ setMethod( "dbReadTable", signature(conn="AthenaConnection", name="character"), - def = function(conn, name, schema, ...) { + definition = function(conn, name, schema, ...) { query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, dbQuoteString(conn, name)) dbGetQuery(conn, query) } diff --git a/R/list-query-executions.R b/R/list-query-executions.R new file mode 100644 index 0000000..115a3cd --- /dev/null +++ b/R/list-query-executions.R @@ -0,0 +1,13 @@ +list_query_executions <- function(max_items=10, starting_token=NULL, page_size=NULL) { + + + args <- c("list-query-executions", sprintf("--max-items=%s", as.integer(max_items))) + + if (!is.null(starting_token)) args <- c(args, sprintf("--starting-token=%s", starting_token)) + if (!is.null(page_size)) args <- c(args, sprintf("--page-size=%s", as.integer(page_size))) + + res <- .athenacli(args) + + jsonlite::fromJSON() + +} \ No newline at end of file diff --git a/R/metis-package.R b/R/metis-package.R index 0acff74..af572b1 100644 --- a/R/metis-package.R +++ b/R/metis-package.R @@ -1,6 +1,8 @@ #' Helpers for Accessing and Querying Amazon Athena #' -#' Including a lightweight RJDBC shim. +#' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, +#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation +#' is included along with an interface to the 'AWS' command-line utility. #' #' @name metis #' @encoding UTF-8 @@ -9,7 +11,10 @@ #' @import RJDBC #' @import DBI #' @import dplyr +#' @importFrom jsonlite fromJSON #' @importFrom readr type_convert +#' @importFrom uuid UUIDgenerate +#' @importFrom sys exec_internal #' @importFrom aws.signature use_credentials read_credentials NULL diff --git a/R/metis.r b/R/metis.r index 7e5af20..64927ed 100644 --- a/R/metis.r +++ b/R/metis.r @@ -44,17 +44,19 @@ athena_connect <- function(default_schema = "default", log_level <- match.arg(log_level, c("INFO", "DEBUG", "WARN", "ERROR", "ALL", "OFF", "FATAL", "TRACE")) # if (!simple) { - con <- dbConnect(athena_jdbc, - schema_name = default_schema, - region = region, - s3_staging_dir = s3_staging_dir, - max_error_retries = max_error_retries, - connection_timeout = connection_timeout, - socket_timeout = socket_timeout, - retry_base_delay = retry_base_delay, - retry_max_backoff_time = retry_max_backoff_time, - log_path = log_path, - log_level = log_level) + dbConnect( + athena_jdbc, + schema_name = default_schema, + region = region, + s3_staging_dir = s3_staging_dir, + max_error_retries = max_error_retries, + connection_timeout = connection_timeout, + socket_timeout = socket_timeout, + retry_base_delay = retry_base_delay, + retry_max_backoff_time = retry_max_backoff_time, + log_path = log_path, + log_level = log_level + ) -> con # } else { # con <- dbConnect(athena_jdbc, provider = NULL, schema_name = default_schema, region = region, # s3_staging_dir = s3_staging_dir, log_path = log_path, log_level = log_level) diff --git a/README.Rmd b/README.Rmd index 10af18c..6b4a8e9 100644 --- a/README.Rmd +++ b/README.Rmd @@ -49,8 +49,8 @@ Custom JDBC Class Methods: Pulled in from other `cloudyr` pkgs: -- `read_credentials` Use Credentials from .aws/credentials File -- `use_credentials` Use Credentials from .aws/credentials File +- `read_credentials`: Use Credentials from .aws/credentials File +- `use_credentials`: Use Credentials from .aws/credentials File ## Installation diff --git a/README.md b/README.md index e9b8eeb..93e6863 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,8 @@ Custom JDBC Class Methods: Pulled in from other `cloudyr` pkgs: - - `read_credentials` Use Credentials from .aws/credentials File - - `use_credentials` Use Credentials from .aws/credentials File + - `read_credentials`: Use Credentials from .aws/credentials File + - `use_credentials`: Use Credentials from .aws/credentials File ## Installation diff --git a/inst/AthenaJDBC41-1.1.0.jar b/inst/java/AthenaJDBC41-1.1.0.jar similarity index 100% rename from inst/AthenaJDBC41-1.1.0.jar rename to inst/java/AthenaJDBC41-1.1.0.jar diff --git a/man/metis.Rd b/man/metis.Rd index faad773..a91bba3 100644 --- a/man/metis.Rd +++ b/man/metis.Rd @@ -7,7 +7,9 @@ \alias{metis-package} \title{Helpers for Accessing and Querying Amazon Athena} \description{ -Including a lightweight RJDBC shim. +Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, +perform queries and retrieve query results. A lightweight 'RJDBC' implementation +is included along with an interface to the 'AWS' command-line utility. } \author{ Bob Rudis (bob@rud.is)