Browse Source

update

master
boB Rudis 6 years ago
parent
commit
b91c192238
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
  1. 14
      DESCRIPTION
  2. 3
      NAMESPACE
  3. 26
      R/awscli-util.R
  4. 55
      R/jdbc.r
  5. 13
      R/list-query-executions.R
  6. 7
      R/metis-package.R
  7. 24
      R/metis.r
  8. 4
      README.Rmd
  9. 4
      README.md
  10. 0
      inst/java/AthenaJDBC41-1.1.0.jar
  11. 4
      man/metis.Rd

14
DESCRIPTION

@ -5,13 +5,14 @@ Version: 0.2.0
Date: 2018-03-19 Date: 2018-03-19
Authors@R: c( Authors@R: c(
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"), person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-5670-2640")), comment = c(ORCID = "0000-0001-5670-2640")),
person("Zachary, "Kurtz", email="zdkurtz@gmail.com", role = "ctb") person("Zachary", "Kurtz", email = "zdkurtz@gmail.com", role = "ctb")
) )
Maintainer: Bob Rudis <bob@rud.is> Maintainer: Bob Rudis <bob@rud.is>
Encoding: UTF-8 Encoding: UTF-8
Description: Helpers for Accessing and Querying Amazon Athena. Including a Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
lightweight RJDBC shim. perform queries and retrieve query results. A lightweight 'RJDBC' implementation
is included along with an interface to the 'AWS' command-line utility.
URL: https://github.com/hrbrmstr/metis URL: https://github.com/hrbrmstr/metis
BugReports: https://github.com/hrbrmstr/metis/issues BugReports: https://github.com/hrbrmstr/metis/issues
License: AGPL License: AGPL
@ -26,5 +27,8 @@ Imports:
DBI, DBI,
dplyr, dplyr,
readr, readr,
aws.signature aws.signature,
uuid,
sys,
jsonlite
RoxygenNote: 6.0.1.9000 RoxygenNote: 6.0.1.9000

3
NAMESPACE

@ -19,5 +19,8 @@ import(RJDBC)
import(dplyr) import(dplyr)
importFrom(aws.signature,read_credentials) importFrom(aws.signature,read_credentials)
importFrom(aws.signature,use_credentials) importFrom(aws.signature,use_credentials)
importFrom(jsonlite,fromJSON)
importFrom(rJava,.jcall) importFrom(rJava,.jcall)
importFrom(readr,type_convert) importFrom(readr,type_convert)
importFrom(sys,exec_internal)
importFrom(uuid,UUIDgenerate)

26
R/awscli-util.R

@ -0,0 +1,26 @@
.aws_bin <- function() {
unname(Sys.which('aws'))
}
.athenacli <- function(...) {
args <- c("athena")
in_args <- list(...)
if (length(in_args) == 0) in_args <- "help"
args <- c(args, unlist(in_args, use.names=FALSE))
res <- sys::exec_internal(.aws_bin(), args = args, error = FALSE)
if (length(res$stdout) > 0) {
out <- rawToChar(res$stdout)
if ("help" %in% args) cat(out, sep="")
invisible(out)
}
}

55
R/jdbc.r

@ -4,18 +4,24 @@
setClass( setClass(
"AthenaDriver", "AthenaDriver",
representation("JDBCDriver", identifier.quote="character", jdrv="jobjRef") representation(
"JDBCDriver",
identifier.quote = "character",
jdrv = "jobjRef"
)
) )
#' AthenaJDBC #' AthenaJDBC
#' #'
#' @export #' @export
Athena <- function(identifier.quote='`') { Athena <- function(identifier.quote = '`') {
drv <- JDBC(driverClass="com.amazonaws.athena.jdbc.AthenaDriver", JDBC(
system.file("AthenaJDBC41-1.1.0.jar", package="metis"), driverClass = "com.amazonaws.athena.jdbc.AthenaDriver",
identifier.quote=identifier.quote) system.file("java", "AthenaJDBC41-1.1.0.jar", package = "metis"),
identifier.quote = identifier.quote
) -> drv
return(as(drv, "AthenaDriver")) return(as(drv, "AthenaDriver"))
@ -57,18 +63,21 @@ setMethod(
'jdbc:awsathena://athena.%s.amazonaws.com:443/%s', region, schema_name 'jdbc:awsathena://athena.%s.amazonaws.com:443/%s', region, schema_name
) )
jc <- callNextMethod(drv, conn_string, callNextMethod(
s3_staging_dir = s3_staging_dir, drv,
schema_name = schema_name, conn_string,
max_error_retries = max_error_retries, s3_staging_dir = s3_staging_dir,
connection_timeout = connection_timeout, schema_name = schema_name,
socket_timeout = socket_timeout, max_error_retries = max_error_retries,
retry_base_delay = retry_base_delay, connection_timeout = connection_timeout,
retry_max_backoff_time = retry_max_backoff_time, socket_timeout = socket_timeout,
log_path = log_path, retry_base_delay = retry_base_delay,
log_level = log_level, retry_max_backoff_time = retry_max_backoff_time,
aws_credentials_provider_class = provider, log_path = log_path,
...) log_level = log_level,
aws_credentials_provider_class = provider,
...
) -> jc
return(as(jc, "AthenaConnection")) return(as(jc, "AthenaConnection"))
@ -97,7 +106,7 @@ setMethod(
"dbSendQuery", "dbSendQuery",
signature(conn="AthenaConnection", statement="character"), signature(conn="AthenaConnection", statement="character"),
def = function(conn, statement, ...) { definition = function(conn, statement, ...) {
return(as(callNextMethod(), "AthenaResult")) return(as(callNextMethod(), "AthenaResult"))
} }
@ -115,7 +124,7 @@ setMethod(
"dbGetQuery", "dbGetQuery",
signature(conn="AthenaConnection", statement="character"), signature(conn="AthenaConnection", statement="character"),
def = function(conn, statement, type_convert=FALSE, ...) { definition = function(conn, statement, type_convert=FALSE, ...) {
r <- dbSendQuery(conn, statement, ...) r <- dbSendQuery(conn, statement, ...)
on.exit(.jcall(r@stat, "V", "close")) on.exit(.jcall(r@stat, "V", "close"))
res <- dplyr::tbl_df(fetch(r, -1, block=1000)) res <- dplyr::tbl_df(fetch(r, -1, block=1000))
@ -136,7 +145,7 @@ setMethod(
"dbListTables", "dbListTables",
signature(conn="AthenaConnection"), signature(conn="AthenaConnection"),
def = function(conn, pattern='*', schema, ...) { definition = function(conn, pattern='*', schema, ...) {
if (missing(pattern)) { if (missing(pattern)) {
dbGetQuery( dbGetQuery(
@ -165,7 +174,7 @@ setMethod(
"dbExistsTable", "dbExistsTable",
signature(conn="AthenaConnection", name="character"), signature(conn="AthenaConnection", name="character"),
def = function(conn, name, schema, ...) { definition = function(conn, name, schema, ...) {
length(dbListTables(conn, schema=schema, pattern=name)) > 0 length(dbListTables(conn, schema=schema, pattern=name)) > 0
} }
@ -183,7 +192,7 @@ setMethod(
"dbListFields", "dbListFields",
signature(conn="AthenaConnection", name="character"), signature(conn="AthenaConnection", name="character"),
def = function(conn, name, schema, ...) { definition = function(conn, name, schema, ...) {
query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, name) query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, name)
res <- dbGetQuery(conn, query) res <- dbGetQuery(conn, query)
colnames(res) colnames(res)
@ -203,7 +212,7 @@ setMethod(
"dbReadTable", "dbReadTable",
signature(conn="AthenaConnection", name="character"), signature(conn="AthenaConnection", name="character"),
def = function(conn, name, schema, ...) { definition = function(conn, name, schema, ...) {
query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, dbQuoteString(conn, name)) query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, dbQuoteString(conn, name))
dbGetQuery(conn, query) dbGetQuery(conn, query)
} }

13
R/list-query-executions.R

@ -0,0 +1,13 @@
list_query_executions <- function(max_items=10, starting_token=NULL, page_size=NULL) {
args <- c("list-query-executions", sprintf("--max-items=%s", as.integer(max_items)))
if (!is.null(starting_token)) args <- c(args, sprintf("--starting-token=%s", starting_token))
if (!is.null(page_size)) args <- c(args, sprintf("--page-size=%s", as.integer(page_size)))
res <- .athenacli(args)
jsonlite::fromJSON()
}

7
R/metis-package.R

@ -1,6 +1,8 @@
#' Helpers for Accessing and Querying Amazon Athena #' Helpers for Accessing and Querying Amazon Athena
#' #'
#' Including a lightweight RJDBC shim. #' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation
#' is included along with an interface to the 'AWS' command-line utility.
#' #'
#' @name metis #' @name metis
#' @encoding UTF-8 #' @encoding UTF-8
@ -9,7 +11,10 @@
#' @import RJDBC #' @import RJDBC
#' @import DBI #' @import DBI
#' @import dplyr #' @import dplyr
#' @importFrom jsonlite fromJSON
#' @importFrom readr type_convert #' @importFrom readr type_convert
#' @importFrom uuid UUIDgenerate
#' @importFrom sys exec_internal
#' @importFrom aws.signature use_credentials read_credentials #' @importFrom aws.signature use_credentials read_credentials
NULL NULL

24
R/metis.r

@ -44,17 +44,19 @@ athena_connect <- function(default_schema = "default",
log_level <- match.arg(log_level, c("INFO", "DEBUG", "WARN", "ERROR", "ALL", "OFF", "FATAL", "TRACE")) log_level <- match.arg(log_level, c("INFO", "DEBUG", "WARN", "ERROR", "ALL", "OFF", "FATAL", "TRACE"))
# if (!simple) { # if (!simple) {
con <- dbConnect(athena_jdbc, dbConnect(
schema_name = default_schema, athena_jdbc,
region = region, schema_name = default_schema,
s3_staging_dir = s3_staging_dir, region = region,
max_error_retries = max_error_retries, s3_staging_dir = s3_staging_dir,
connection_timeout = connection_timeout, max_error_retries = max_error_retries,
socket_timeout = socket_timeout, connection_timeout = connection_timeout,
retry_base_delay = retry_base_delay, socket_timeout = socket_timeout,
retry_max_backoff_time = retry_max_backoff_time, retry_base_delay = retry_base_delay,
log_path = log_path, retry_max_backoff_time = retry_max_backoff_time,
log_level = log_level) log_path = log_path,
log_level = log_level
) -> con
# } else { # } else {
# con <- dbConnect(athena_jdbc, provider = NULL, schema_name = default_schema, region = region, # con <- dbConnect(athena_jdbc, provider = NULL, schema_name = default_schema, region = region,
# s3_staging_dir = s3_staging_dir, log_path = log_path, log_level = log_level) # s3_staging_dir = s3_staging_dir, log_path = log_path, log_level = log_level)

4
README.Rmd

@ -49,8 +49,8 @@ Custom JDBC Class Methods:
Pulled in from other `cloudyr` pkgs: Pulled in from other `cloudyr` pkgs:
- `read_credentials` Use Credentials from .aws/credentials File - `read_credentials`: Use Credentials from .aws/credentials File
- `use_credentials` Use Credentials from .aws/credentials File - `use_credentials`: Use Credentials from .aws/credentials File
## Installation ## Installation

4
README.md

@ -52,8 +52,8 @@ Custom JDBC Class Methods:
Pulled in from other `cloudyr` pkgs: Pulled in from other `cloudyr` pkgs:
- `read_credentials` Use Credentials from .aws/credentials File - `read_credentials`: Use Credentials from .aws/credentials File
- `use_credentials` Use Credentials from .aws/credentials File - `use_credentials`: Use Credentials from .aws/credentials File
## Installation ## Installation

0
inst/AthenaJDBC41-1.1.0.jar → inst/java/AthenaJDBC41-1.1.0.jar

4
man/metis.Rd

@ -7,7 +7,9 @@
\alias{metis-package} \alias{metis-package}
\title{Helpers for Accessing and Querying Amazon Athena} \title{Helpers for Accessing and Querying Amazon Athena}
\description{ \description{
Including a lightweight RJDBC shim. Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
perform queries and retrieve query results. A lightweight 'RJDBC' implementation
is included along with an interface to the 'AWS' command-line utility.
} }
\author{ \author{
Bob Rudis (bob@rud.is) Bob Rudis (bob@rud.is)

Loading…
Cancel
Save