boB Rudis 6 anos atrás
pai
commit
b91c192238
Nenhuma chave conhecida encontrada para esta assinatura no banco de dados ID da chave GPG: 1D7529BE14E2BBA9
  1. 14
      DESCRIPTION
  2. 3
      NAMESPACE
  3. 26
      R/awscli-util.R
  4. 55
      R/jdbc.r
  5. 13
      R/list-query-executions.R
  6. 7
      R/metis-package.R
  7. 24
      R/metis.r
  8. 4
      README.Rmd
  9. 4
      README.md
  10. 0
      inst/java/AthenaJDBC41-1.1.0.jar
  11. 4
      man/metis.Rd

14
DESCRIPTION

@ -5,13 +5,14 @@ Version: 0.2.0
Date: 2018-03-19
Authors@R: c(
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-5670-2640")),
person("Zachary, "Kurtz", email="zdkurtz@gmail.com", role = "ctb")
comment = c(ORCID = "0000-0001-5670-2640")),
person("Zachary", "Kurtz", email = "zdkurtz@gmail.com", role = "ctb")
)
Maintainer: Bob Rudis <bob@rud.is>
Encoding: UTF-8
Description: Helpers for Accessing and Querying Amazon Athena. Including a
lightweight RJDBC shim.
Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
perform queries and retrieve query results. A lightweight 'RJDBC' implementation
is included along with an interface to the 'AWS' command-line utility.
URL: https://github.com/hrbrmstr/metis
BugReports: https://github.com/hrbrmstr/metis/issues
License: AGPL
@ -26,5 +27,8 @@ Imports:
DBI,
dplyr,
readr,
aws.signature
aws.signature,
uuid,
sys,
jsonlite
RoxygenNote: 6.0.1.9000

3
NAMESPACE

@ -19,5 +19,8 @@ import(RJDBC)
import(dplyr)
importFrom(aws.signature,read_credentials)
importFrom(aws.signature,use_credentials)
importFrom(jsonlite,fromJSON)
importFrom(rJava,.jcall)
importFrom(readr,type_convert)
importFrom(sys,exec_internal)
importFrom(uuid,UUIDgenerate)

26
R/awscli-util.R

@ -0,0 +1,26 @@
.aws_bin <- function() {
unname(Sys.which('aws'))
}
.athenacli <- function(...) {
args <- c("athena")
in_args <- list(...)
if (length(in_args) == 0) in_args <- "help"
args <- c(args, unlist(in_args, use.names=FALSE))
res <- sys::exec_internal(.aws_bin(), args = args, error = FALSE)
if (length(res$stdout) > 0) {
out <- rawToChar(res$stdout)
if ("help" %in% args) cat(out, sep="")
invisible(out)
}
}

55
R/jdbc.r

@ -4,18 +4,24 @@
setClass(
"AthenaDriver",
representation("JDBCDriver", identifier.quote="character", jdrv="jobjRef")
representation(
"JDBCDriver",
identifier.quote = "character",
jdrv = "jobjRef"
)
)
#' AthenaJDBC
#'
#' @export
Athena <- function(identifier.quote='`') {
Athena <- function(identifier.quote = '`') {
drv <- JDBC(driverClass="com.amazonaws.athena.jdbc.AthenaDriver",
system.file("AthenaJDBC41-1.1.0.jar", package="metis"),
identifier.quote=identifier.quote)
JDBC(
driverClass = "com.amazonaws.athena.jdbc.AthenaDriver",
system.file("java", "AthenaJDBC41-1.1.0.jar", package = "metis"),
identifier.quote = identifier.quote
) -> drv
return(as(drv, "AthenaDriver"))
@ -57,18 +63,21 @@ setMethod(
'jdbc:awsathena://athena.%s.amazonaws.com:443/%s', region, schema_name
)
jc <- callNextMethod(drv, conn_string,
s3_staging_dir = s3_staging_dir,
schema_name = schema_name,
max_error_retries = max_error_retries,
connection_timeout = connection_timeout,
socket_timeout = socket_timeout,
retry_base_delay = retry_base_delay,
retry_max_backoff_time = retry_max_backoff_time,
log_path = log_path,
log_level = log_level,
aws_credentials_provider_class = provider,
...)
callNextMethod(
drv,
conn_string,
s3_staging_dir = s3_staging_dir,
schema_name = schema_name,
max_error_retries = max_error_retries,
connection_timeout = connection_timeout,
socket_timeout = socket_timeout,
retry_base_delay = retry_base_delay,
retry_max_backoff_time = retry_max_backoff_time,
log_path = log_path,
log_level = log_level,
aws_credentials_provider_class = provider,
...
) -> jc
return(as(jc, "AthenaConnection"))
@ -97,7 +106,7 @@ setMethod(
"dbSendQuery",
signature(conn="AthenaConnection", statement="character"),
def = function(conn, statement, ...) {
definition = function(conn, statement, ...) {
return(as(callNextMethod(), "AthenaResult"))
}
@ -115,7 +124,7 @@ setMethod(
"dbGetQuery",
signature(conn="AthenaConnection", statement="character"),
def = function(conn, statement, type_convert=FALSE, ...) {
definition = function(conn, statement, type_convert=FALSE, ...) {
r <- dbSendQuery(conn, statement, ...)
on.exit(.jcall(r@stat, "V", "close"))
res <- dplyr::tbl_df(fetch(r, -1, block=1000))
@ -136,7 +145,7 @@ setMethod(
"dbListTables",
signature(conn="AthenaConnection"),
def = function(conn, pattern='*', schema, ...) {
definition = function(conn, pattern='*', schema, ...) {
if (missing(pattern)) {
dbGetQuery(
@ -165,7 +174,7 @@ setMethod(
"dbExistsTable",
signature(conn="AthenaConnection", name="character"),
def = function(conn, name, schema, ...) {
definition = function(conn, name, schema, ...) {
length(dbListTables(conn, schema=schema, pattern=name)) > 0
}
@ -183,7 +192,7 @@ setMethod(
"dbListFields",
signature(conn="AthenaConnection", name="character"),
def = function(conn, name, schema, ...) {
definition = function(conn, name, schema, ...) {
query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, name)
res <- dbGetQuery(conn, query)
colnames(res)
@ -203,7 +212,7 @@ setMethod(
"dbReadTable",
signature(conn="AthenaConnection", name="character"),
def = function(conn, name, schema, ...) {
definition = function(conn, name, schema, ...) {
query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, dbQuoteString(conn, name))
dbGetQuery(conn, query)
}

13
R/list-query-executions.R

@ -0,0 +1,13 @@
list_query_executions <- function(max_items=10, starting_token=NULL, page_size=NULL) {
args <- c("list-query-executions", sprintf("--max-items=%s", as.integer(max_items)))
if (!is.null(starting_token)) args <- c(args, sprintf("--starting-token=%s", starting_token))
if (!is.null(page_size)) args <- c(args, sprintf("--page-size=%s", as.integer(page_size)))
res <- .athenacli(args)
jsonlite::fromJSON()
}

7
R/metis-package.R

@ -1,6 +1,8 @@
#' Helpers for Accessing and Querying Amazon Athena
#'
#' Including a lightweight RJDBC shim.
#' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation
#' is included along with an interface to the 'AWS' command-line utility.
#'
#' @name metis
#' @encoding UTF-8
@ -9,7 +11,10 @@
#' @import RJDBC
#' @import DBI
#' @import dplyr
#' @importFrom jsonlite fromJSON
#' @importFrom readr type_convert
#' @importFrom uuid UUIDgenerate
#' @importFrom sys exec_internal
#' @importFrom aws.signature use_credentials read_credentials
NULL

24
R/metis.r

@ -44,17 +44,19 @@ athena_connect <- function(default_schema = "default",
log_level <- match.arg(log_level, c("INFO", "DEBUG", "WARN", "ERROR", "ALL", "OFF", "FATAL", "TRACE"))
# if (!simple) {
con <- dbConnect(athena_jdbc,
schema_name = default_schema,
region = region,
s3_staging_dir = s3_staging_dir,
max_error_retries = max_error_retries,
connection_timeout = connection_timeout,
socket_timeout = socket_timeout,
retry_base_delay = retry_base_delay,
retry_max_backoff_time = retry_max_backoff_time,
log_path = log_path,
log_level = log_level)
dbConnect(
athena_jdbc,
schema_name = default_schema,
region = region,
s3_staging_dir = s3_staging_dir,
max_error_retries = max_error_retries,
connection_timeout = connection_timeout,
socket_timeout = socket_timeout,
retry_base_delay = retry_base_delay,
retry_max_backoff_time = retry_max_backoff_time,
log_path = log_path,
log_level = log_level
) -> con
# } else {
# con <- dbConnect(athena_jdbc, provider = NULL, schema_name = default_schema, region = region,
# s3_staging_dir = s3_staging_dir, log_path = log_path, log_level = log_level)

4
README.Rmd

@ -49,8 +49,8 @@ Custom JDBC Class Methods:
Pulled in from other `cloudyr` pkgs:
- `read_credentials` Use Credentials from .aws/credentials File
- `use_credentials` Use Credentials from .aws/credentials File
- `read_credentials`: Use Credentials from .aws/credentials File
- `use_credentials`: Use Credentials from .aws/credentials File
## Installation

4
README.md

@ -52,8 +52,8 @@ Custom JDBC Class Methods:
Pulled in from other `cloudyr` pkgs:
- `read_credentials` Use Credentials from .aws/credentials File
- `use_credentials` Use Credentials from .aws/credentials File
- `read_credentials`: Use Credentials from .aws/credentials File
- `use_credentials`: Use Credentials from .aws/credentials File
## Installation

0
inst/AthenaJDBC41-1.1.0.jar → inst/java/AthenaJDBC41-1.1.0.jar

4
man/metis.Rd

@ -7,7 +7,9 @@
\alias{metis-package}
\title{Helpers for Accessing and Querying Amazon Athena}
\description{
Including a lightweight RJDBC shim.
Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
perform queries and retrieve query results. A lightweight 'RJDBC' implementation
is included along with an interface to the 'AWS' command-line utility.
}
\author{
Bob Rudis (bob@rud.is)

Carregando…
Cancelar
Salvar