Browse Source

update

master
boB Rudis 2 years ago
parent
commit
b91c192238
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
11 changed files with 109 additions and 45 deletions
  1. +9
    -5
      DESCRIPTION
  2. +3
    -0
      NAMESPACE
  3. +26
    -0
      R/awscli-util.R
  4. +32
    -23
      R/jdbc.r
  5. +13
    -0
      R/list-query-executions.R
  6. +6
    -1
      R/metis-package.R
  7. +13
    -11
      R/metis.r
  8. +2
    -2
      README.Rmd
  9. +2
    -2
      README.md
  10. +0
    -0
      inst/java/AthenaJDBC41-1.1.0.jar
  11. +3
    -1
      man/metis.Rd

+ 9
- 5
DESCRIPTION View File

@@ -5,13 +5,14 @@ Version: 0.2.0
Date: 2018-03-19
Authors@R: c(
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-5670-2640")),
person("Zachary, "Kurtz", email="zdkurtz@gmail.com", role = "ctb")
comment = c(ORCID = "0000-0001-5670-2640")),
person("Zachary", "Kurtz", email = "zdkurtz@gmail.com", role = "ctb")
)
Maintainer: Bob Rudis <bob@rud.is>
Encoding: UTF-8
Description: Helpers for Accessing and Querying Amazon Athena. Including a
lightweight RJDBC shim.
Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
perform queries and retrieve query results. A lightweight 'RJDBC' implementation
is included along with an interface to the 'AWS' command-line utility.
URL: https://github.com/hrbrmstr/metis
BugReports: https://github.com/hrbrmstr/metis/issues
License: AGPL
@@ -26,5 +27,8 @@ Imports:
DBI,
dplyr,
readr,
aws.signature
aws.signature,
uuid,
sys,
jsonlite
RoxygenNote: 6.0.1.9000

+ 3
- 0
NAMESPACE View File

@@ -19,5 +19,8 @@ import(RJDBC)
import(dplyr)
importFrom(aws.signature,read_credentials)
importFrom(aws.signature,use_credentials)
importFrom(jsonlite,fromJSON)
importFrom(rJava,.jcall)
importFrom(readr,type_convert)
importFrom(sys,exec_internal)
importFrom(uuid,UUIDgenerate)

+ 26
- 0
R/awscli-util.R View File

@@ -0,0 +1,26 @@
.aws_bin <- function() {
unname(Sys.which('aws'))
}

.athenacli <- function(...) {

args <- c("athena")

in_args <- list(...)
if (length(in_args) == 0) in_args <- "help"

args <- c(args, unlist(in_args, use.names=FALSE))

res <- sys::exec_internal(.aws_bin(), args = args, error = FALSE)

if (length(res$stdout) > 0) {

out <- rawToChar(res$stdout)

if ("help" %in% args) cat(out, sep="")

invisible(out)

}

}

+ 32
- 23
R/jdbc.r View File

@@ -4,18 +4,24 @@
setClass(

"AthenaDriver",
representation("JDBCDriver", identifier.quote="character", jdrv="jobjRef")
representation(
"JDBCDriver",
identifier.quote = "character",
jdrv = "jobjRef"
)

)

#' AthenaJDBC
#'
#' @export
Athena <- function(identifier.quote='`') {
Athena <- function(identifier.quote = '`') {

drv <- JDBC(driverClass="com.amazonaws.athena.jdbc.AthenaDriver",
system.file("AthenaJDBC41-1.1.0.jar", package="metis"),
identifier.quote=identifier.quote)
JDBC(
driverClass = "com.amazonaws.athena.jdbc.AthenaDriver",
system.file("java", "AthenaJDBC41-1.1.0.jar", package = "metis"),
identifier.quote = identifier.quote
) -> drv

return(as(drv, "AthenaDriver"))

@@ -57,18 +63,21 @@ setMethod(
'jdbc:awsathena://athena.%s.amazonaws.com:443/%s', region, schema_name
)

jc <- callNextMethod(drv, conn_string,
s3_staging_dir = s3_staging_dir,
schema_name = schema_name,
max_error_retries = max_error_retries,
connection_timeout = connection_timeout,
socket_timeout = socket_timeout,
retry_base_delay = retry_base_delay,
retry_max_backoff_time = retry_max_backoff_time,
log_path = log_path,
log_level = log_level,
aws_credentials_provider_class = provider,
...)
callNextMethod(
drv,
conn_string,
s3_staging_dir = s3_staging_dir,
schema_name = schema_name,
max_error_retries = max_error_retries,
connection_timeout = connection_timeout,
socket_timeout = socket_timeout,
retry_base_delay = retry_base_delay,
retry_max_backoff_time = retry_max_backoff_time,
log_path = log_path,
log_level = log_level,
aws_credentials_provider_class = provider,
...
) -> jc

return(as(jc, "AthenaConnection"))

@@ -97,7 +106,7 @@ setMethod(
"dbSendQuery",
signature(conn="AthenaConnection", statement="character"),

def = function(conn, statement, ...) {
definition = function(conn, statement, ...) {
return(as(callNextMethod(), "AthenaResult"))
}

@@ -115,7 +124,7 @@ setMethod(
"dbGetQuery",
signature(conn="AthenaConnection", statement="character"),

def = function(conn, statement, type_convert=FALSE, ...) {
definition = function(conn, statement, type_convert=FALSE, ...) {
r <- dbSendQuery(conn, statement, ...)
on.exit(.jcall(r@stat, "V", "close"))
res <- dplyr::tbl_df(fetch(r, -1, block=1000))
@@ -136,7 +145,7 @@ setMethod(
"dbListTables",
signature(conn="AthenaConnection"),

def = function(conn, pattern='*', schema, ...) {
definition = function(conn, pattern='*', schema, ...) {

if (missing(pattern)) {
dbGetQuery(
@@ -165,7 +174,7 @@ setMethod(
"dbExistsTable",
signature(conn="AthenaConnection", name="character"),

def = function(conn, name, schema, ...) {
definition = function(conn, name, schema, ...) {
length(dbListTables(conn, schema=schema, pattern=name)) > 0
}

@@ -183,7 +192,7 @@ setMethod(
"dbListFields",
signature(conn="AthenaConnection", name="character"),

def = function(conn, name, schema, ...) {
definition = function(conn, name, schema, ...) {
query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, name)
res <- dbGetQuery(conn, query)
colnames(res)
@@ -203,7 +212,7 @@ setMethod(
"dbReadTable",
signature(conn="AthenaConnection", name="character"),

def = function(conn, name, schema, ...) {
definition = function(conn, name, schema, ...) {
query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, dbQuoteString(conn, name))
dbGetQuery(conn, query)
}


+ 13
- 0
R/list-query-executions.R View File

@@ -0,0 +1,13 @@
list_query_executions <- function(max_items=10, starting_token=NULL, page_size=NULL) {


args <- c("list-query-executions", sprintf("--max-items=%s", as.integer(max_items)))

if (!is.null(starting_token)) args <- c(args, sprintf("--starting-token=%s", starting_token))
if (!is.null(page_size)) args <- c(args, sprintf("--page-size=%s", as.integer(page_size)))

res <- .athenacli(args)

jsonlite::fromJSON()

}

+ 6
- 1
R/metis-package.R View File

@@ -1,6 +1,8 @@
#' Helpers for Accessing and Querying Amazon Athena
#'
#' Including a lightweight RJDBC shim.
#' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation
#' is included along with an interface to the 'AWS' command-line utility.
#'
#' @name metis
#' @encoding UTF-8
@@ -9,7 +11,10 @@
#' @import RJDBC
#' @import DBI
#' @import dplyr
#' @importFrom jsonlite fromJSON
#' @importFrom readr type_convert
#' @importFrom uuid UUIDgenerate
#' @importFrom sys exec_internal
#' @importFrom aws.signature use_credentials read_credentials
NULL



+ 13
- 11
R/metis.r View File

@@ -44,17 +44,19 @@ athena_connect <- function(default_schema = "default",
log_level <- match.arg(log_level, c("INFO", "DEBUG", "WARN", "ERROR", "ALL", "OFF", "FATAL", "TRACE"))

# if (!simple) {
con <- dbConnect(athena_jdbc,
schema_name = default_schema,
region = region,
s3_staging_dir = s3_staging_dir,
max_error_retries = max_error_retries,
connection_timeout = connection_timeout,
socket_timeout = socket_timeout,
retry_base_delay = retry_base_delay,
retry_max_backoff_time = retry_max_backoff_time,
log_path = log_path,
log_level = log_level)
dbConnect(
athena_jdbc,
schema_name = default_schema,
region = region,
s3_staging_dir = s3_staging_dir,
max_error_retries = max_error_retries,
connection_timeout = connection_timeout,
socket_timeout = socket_timeout,
retry_base_delay = retry_base_delay,
retry_max_backoff_time = retry_max_backoff_time,
log_path = log_path,
log_level = log_level
) -> con
# } else {
# con <- dbConnect(athena_jdbc, provider = NULL, schema_name = default_schema, region = region,
# s3_staging_dir = s3_staging_dir, log_path = log_path, log_level = log_level)


+ 2
- 2
README.Rmd View File

@@ -49,8 +49,8 @@ Custom JDBC Class Methods:

Pulled in from other `cloudyr` pkgs:

- `read_credentials` Use Credentials from .aws/credentials File
- `use_credentials` Use Credentials from .aws/credentials File
- `read_credentials`: Use Credentials from .aws/credentials File
- `use_credentials`: Use Credentials from .aws/credentials File

## Installation



+ 2
- 2
README.md View File

@@ -52,8 +52,8 @@ Custom JDBC Class Methods:

Pulled in from other `cloudyr` pkgs:

- `read_credentials` Use Credentials from .aws/credentials File
- `use_credentials` Use Credentials from .aws/credentials File
- `read_credentials`: Use Credentials from .aws/credentials File
- `use_credentials`: Use Credentials from .aws/credentials File

## Installation



inst/AthenaJDBC41-1.1.0.jar → inst/java/AthenaJDBC41-1.1.0.jar View File


+ 3
- 1
man/metis.Rd View File

@@ -7,7 +7,9 @@
\alias{metis-package}
\title{Helpers for Accessing and Querying Amazon Athena}
\description{
Including a lightweight RJDBC shim.
Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
perform queries and retrieve query results. A lightweight 'RJDBC' implementation
is included along with an interface to the 'AWS' command-line utility.
}
\author{
Bob Rudis (bob@rud.is)


Loading…
Cancel
Save