mirror of https://git.sr.ht/~hrbrmstr/metis-tidy
34 changed files with 230 additions and 1114 deletions
@ -1,27 +1,8 @@ |
|||
# Generated by roxygen2: do not edit by hand |
|||
|
|||
export(Athena) |
|||
export(athena_connect) |
|||
export(db_data_type.AthenaConnection) |
|||
export(read_credentials) |
|||
export(sql_translate_env.AthenaConnection) |
|||
export(use_credentials) |
|||
exportClasses(AthenaConnection) |
|||
exportClasses(AthenaDriver) |
|||
exportClasses(AthenaResult) |
|||
exportMethods(dbConnect) |
|||
exportMethods(dbExistsTable) |
|||
exportMethods(dbGetInfo) |
|||
exportMethods(dbGetQuery) |
|||
exportMethods(dbListFields) |
|||
exportMethods(dbListTables) |
|||
exportMethods(dbReadTable) |
|||
exportMethods(dbSendQuery) |
|||
exportMethods(fetch) |
|||
import(DBI) |
|||
import(RJDBC) |
|||
import(bit64) |
|||
import(dbplyr) |
|||
importFrom(aws.signature,read_credentials) |
|||
importFrom(aws.signature,use_credentials) |
|||
importFrom(rJava,.jcall) |
|||
import(metis) |
|||
importFrom(dplyr,tbl) |
|||
|
|||
@ -1,223 +0,0 @@ |
|||
structure( |
|||
0:6, |
|||
.Names = c( |
|||
"OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE" |
|||
) |
|||
)-> .ll_trans |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @export |
|||
setClass( |
|||
|
|||
"AthenaDriver", |
|||
representation( |
|||
"JDBCDriver", |
|||
identifier.quote = "character", |
|||
jdrv = "jobjRef" |
|||
) |
|||
|
|||
) |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @export |
|||
Athena <- function(identifier.quote = '`') { |
|||
|
|||
JDBC( |
|||
driverClass = "com.simba.athena.jdbc.Driver", |
|||
system.file("java", "AthenaJDBC42_2.0.6.jar", package = "metis.lite"), |
|||
identifier.quote = identifier.quote |
|||
) -> drv |
|||
|
|||
return(as(drv, "AthenaDriver")) |
|||
|
|||
} |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' Connect to Athena |
|||
#' |
|||
#' @section Driver Configuration Options: |
|||
#' |
|||
#' - `BinaryColumnLength`: <int> The maximum data length for `BINARY` columns. Default `32767L` |
|||
#' - `ComplexTypeColumnLength`: <int> The maximum data length for `ARRAY`, `MAP`, and `STRUCT` columns. Default `65535L` |
|||
#' - `StringColumnLength`: <int> The maximum data length for `STRING` columns. Default `255L` |
|||
#' |
|||
#' @param provider JDBC auth provider (ideally leave default) |
|||
#' @param region AWS region the Athena tables are in |
|||
#' @param s3_staging_dir A write-able bucket on S3 that you have permissions for |
|||
#' @param schema_name LOL if only this actually worked with Amazon's hacked Presto driver |
|||
#' @param max_error_retries,connection_timeout,socket_timeout |
|||
#' technical connection info that you should only muck with if you know what you're doing. |
|||
#' @param log_path,log_level The Athena JDBC driver can (shockingly) provide a decent bit |
|||
#' of data in logs. Set this to a temporary directory or something log4j can use. For |
|||
#' `log_level` use the names ("INFO", "DEBUG", "WARN", "ERROR", "ALL", "OFF", "FATAL", "TRACE") or |
|||
#' their corresponding integer values 0-6. |
|||
#' @param ... passed on to the driver. See Details. |
|||
#' @references [Connect with JDBC](https://docs.aws.amazon.com/athena/latest/ug/connect-with-jdbc.html); |
|||
#' [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf) |
|||
#' @export |
|||
setMethod( |
|||
|
|||
"dbConnect", |
|||
"AthenaDriver", |
|||
|
|||
def = function( |
|||
drv, |
|||
provider = "com.simba.athena.amazonaws.auth.DefaultAWSCredentialsProviderChain", |
|||
region = "us-east-1", |
|||
s3_staging_dir = Sys.getenv("AWS_S3_STAGING_DIR"), |
|||
schema_name = "default", |
|||
fetch_size = 1000L, |
|||
max_error_retries = 10, |
|||
connection_timeout = 10000, |
|||
socket_timeout = 10000, |
|||
log_path = "", |
|||
log_level = 0, |
|||
...) { |
|||
|
|||
conn_string = sprintf( |
|||
'jdbc:awsathena://athena.%s.amazonaws.com:443/%s', region, schema_name |
|||
) |
|||
|
|||
if (!(log_level %in% 0:6)) log_level <- .ll_trans[log_level] |
|||
|
|||
callNextMethod( |
|||
drv, |
|||
conn_string, |
|||
S3OutputLocation = s3_staging_dir, |
|||
Schema = schema_name, |
|||
MaxErrorRetry = max_error_retries, |
|||
ConnectTimeout = connection_timeout, |
|||
SocketTimeout = socket_timeout, |
|||
LogPath = log_path, |
|||
LogLevel = log_level, |
|||
AwsCredentialsProviderClass = provider, |
|||
... |
|||
) -> jc |
|||
|
|||
|
|||
jc <- as(jc, "AthenaConnection") |
|||
jc@fetch_size <- as.integer(fetch_size) |
|||
|
|||
return(jc) |
|||
|
|||
} |
|||
|
|||
) |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @export |
|||
setClass("AthenaConnection", representation("JDBCConnection", jc="jobjRef", identifier.quote="character", fetch_size="integer")) |
|||
|
|||
# setClass("AthenaConnection", contains = "JDBCConnection") |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @export |
|||
setClass("AthenaResult", contains = "JDBCResult") |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @param conn Athena connection |
|||
#' @param statement SQL statement |
|||
#' @param ... unused |
|||
#' @export |
|||
setMethod( |
|||
|
|||
"dbSendQuery", |
|||
signature(conn="AthenaConnection", statement="character"), |
|||
|
|||
definition = function(conn, statement, ...) { |
|||
return(as(callNextMethod(), "AthenaResult")) |
|||
} |
|||
|
|||
) |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @param conn Athena connection |
|||
#' @param pattern table name pattern |
|||
#' @param schema Athena schema name |
|||
#' @param ... unused |
|||
#' @export |
|||
setMethod( |
|||
|
|||
"dbListTables", |
|||
signature(conn="AthenaConnection"), |
|||
|
|||
definition = function(conn, pattern='*', schema, ...) { |
|||
|
|||
if (missing(pattern)) { |
|||
dbGetQuery( |
|||
conn, sprintf("SHOW TABLES IN %s", schema) |
|||
) -> x |
|||
} else { |
|||
dbGetQuery( |
|||
conn, sprintf("SHOW TABLES IN %s %s", schema, dbQuoteString(conn, pattern)) |
|||
) -> x |
|||
} |
|||
|
|||
x$tab_name |
|||
} |
|||
|
|||
) |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @param conn Athena connection |
|||
#' @param name table name |
|||
#' @param schema Athena schema name |
|||
#' @param ... unused |
|||
#' @export |
|||
setMethod( |
|||
|
|||
"dbExistsTable", |
|||
signature(conn="AthenaConnection", name="character"), |
|||
|
|||
definition = function(conn, name, schema, ...) { |
|||
length(dbListTables(conn, schema=schema, pattern=name)) > 0 |
|||
} |
|||
|
|||
) |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @param conn Athena connection |
|||
#' @param name table name |
|||
#' @param schema Athena schema name |
|||
#' @param ... unused |
|||
#' @export |
|||
setMethod( |
|||
|
|||
"dbListFields", |
|||
signature(conn="AthenaConnection", name="character"), |
|||
|
|||
definition = function(conn, name, schema, ...) { |
|||
query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, name) |
|||
res <- dbGetQuery(conn, query) |
|||
colnames(res) |
|||
} |
|||
|
|||
) |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @param conn Athena connection |
|||
#' @param name table name |
|||
#' @param schema Athena schema name |
|||
#' @param ... unused |
|||
#' @export |
|||
setMethod( |
|||
|
|||
"dbReadTable", |
|||
signature(conn="AthenaConnection", name="character"), |
|||
|
|||
definition = function(conn, name, schema, ...) { |
|||
query <- sprintf("SELECT * FROM %s.%s LIMIT 1", schema, dbQuoteString(conn, name)) |
|||
dbGetQuery(conn, query) |
|||
} |
|||
|
|||
) |
|||
@ -1,48 +0,0 @@ |
|||
#' Access and Query Amazon Athena via DBI/JDBC |
|||
#' |
|||
#' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, |
|||
#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation |
|||
#' is included along with an interface to the 'AWS' command-line utility. |
|||
#' |
|||
#' @name metis.lite |
|||
#' |
|||
#' @section IMPORTANT: |
|||
#' |
|||
#' Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting |
|||
#' Athena JDBC calls crash the R #' interpreter. You need to set the `-Xrs` option to avoid |
|||
#' signals being passed on to the JVM owner. That has to be done _before_ `rJava` is |
|||
#' loaded so you either need to remember to put it at the top of all scripts _or_ stick this |
|||
#' in your local #' `~/.Rprofile` and/or sitewide `Rprofile`: |
|||
#' |
|||
#' |
|||
#' if (!grepl("-Xrs", getOption("java.parameters", ""))) { |
|||
#' options( |
|||
#' "java.parameters" = c(getOption("java.parameters", default = NULL), "-Xrs") |
|||
#' ) |
|||
#' } |
|||
#' |
|||
#' @md |
|||
#' @encoding UTF-8 |
|||
#' @keywords internal |
|||
#' @docType package |
|||
#' @author Bob Rudis (bob@@rud.is) |
|||
#' @import RJDBC DBI bit64 dbplyr |
|||
#' @references [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf) |
|||
NULL |
|||
|
|||
|
|||
#' Use Credentials from .aws/credentials File |
|||
#' |
|||
#' @md |
|||
#' @importFrom aws.signature use_credentials read_credentials |
|||
#' @references [aws.signature::use_credentials()] / [aws.signature::read_credentials()] |
|||
#' @name use_credentials |
|||
#' @rdname use_credentials |
|||
#' @inheritParams aws.signature::use_credentials |
|||
#' @export |
|||
NULL |
|||
|
|||
#' @name read_credentials |
|||
#' @rdname use_credentials |
|||
#' @export |
|||
NULL |
|||
@ -0,0 +1,15 @@ |
|||
#' Access and Query Amazon Athena via the Tidyverse |
|||
#' |
|||
#' Methods are provided to use the 'metis' JDBC/DBI interface via |
|||
#' the Tidyverse (e.g. 'dbplyr'/'dplyr' idioms). |
|||
#' |
|||
#' @md |
|||
#' @name metis.tidy |
|||
#' @encoding UTF-8 |
|||
#' @keywords internal |
|||
#' @docType package |
|||
#' @author Bob Rudis (bob@@rud.is) |
|||
#' @import metis DBI dbplyr |
|||
#' @importFrom dplyr tbl |
|||
#' @references [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf) |
|||
NULL |
|||
@ -1,68 +0,0 @@ |
|||
#' Simplified Athena JDBC connection helper |
|||
#' |
|||
#' Handles the up-front JDBC config |
|||
#' |
|||
#' @md |
|||
#' @param default_schema default schema (you'll still need to fully qualify non-default schema table names) |
|||
#' @param region AWS region (Ref: <http://docs.aws.amazon.com/general/latest/gr/rande.html#athena>) |
|||
#' @param s3_staging_dir the Amazon S3 location to which your query output is written. The JDBC driver then asks Athena to read the results and provide rows of data back to the user. |
|||
#' @param max_error_retries the maximum number of retries that the JDBC client attempts to make a request to Athena. |
|||
#' @param connection_timeout the maximum amount of time, in milliseconds, to make a successful connection to Athena before an attempt is terminated. |
|||
#' @param socket_timeout the maximum amount of time, in milliseconds, to wait for a socket in order to send data to Athena. |
|||
# @param retry_base_delay minimum delay amount, in milliseconds, between retrying attempts to connect Athena. |
|||
# @param retry_max_backoff_time maximum delay amount, in milliseconds, between retrying attempts to connect Athena. |
|||
#' @param log_path local path of the Athena JDBC driver logs. If no log path is provided, then no log files are created. |
|||
#' @param log_level log level of the Athena JDBC driver logs. Use names |
|||
#' "OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE". |
|||
#' @param ... passed on to the driver |
|||
#' @export |
|||
#' @references [Connect with JDBC](https://docs.aws.amazon.com/athena/latest/ug/connect-with-jdbc.html); |
|||
#' [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf) |
|||
#' @examples \dontrun{ |
|||
#' use_credentials("personal") |
|||
#' |
|||
#' athena_connect( |
|||
#' default_schema = "sampledb", |
|||
#' s3_staging_dir = "s3://accessible-bucket", |
|||
#' log_path = "/tmp/athena.log", |
|||
#' log_level = "DEBUG" |
|||
#' ) -> ath |
|||
#' |
|||
#' dbListTables(ath) |
|||
#' |
|||
#' dbGetQuery(ath, "SELECT * FROM sampledb.elb_logs LIMIT 1") |
|||
#' |
|||
#' } |
|||
athena_connect <- function( |
|||
default_schema = "default", |
|||
region = c("us-east-1", "us-east-2", "us-west-2"), |
|||
s3_staging_dir = Sys.getenv("AWS_S3_STAGING_DIR"), |
|||
max_error_retries = 10, |
|||
connection_timeout = 10000, |
|||
socket_timeout = 10000, |
|||
log_path = "", |
|||
log_level = c("OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE"), |
|||
... |
|||
) { |
|||
|
|||
athena_jdbc <- Athena() |
|||
|
|||
region <- match.arg(region, c("us-east-1", "us-east-2", "us-west-2")) |
|||
log_level <- match.arg(log_level, c("OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE")) |
|||
|
|||
dbConnect( |
|||
athena_jdbc, |
|||
schema_name = default_schema, |
|||
region = region, |
|||
s3_staging_dir = s3_staging_dir, |
|||
max_error_retries = max_error_retries, |
|||
connection_timeout = connection_timeout, |
|||
socket_timeout = socket_timeout, |
|||
log_path = log_path, |
|||
log_level = log_level, |
|||
... |
|||
) -> con |
|||
|
|||
con |
|||
|
|||
} |
|||
@ -1,121 +0,0 @@ |
|||
list( |
|||
"-7" = as.logical, # BIT |
|||
"-6" = as.integer, # TINYINT |
|||
"-5" = bit64::as.integer64, # BIGINT |
|||
"-4" = as.character, # LONGVARBINARY |
|||
"-3" = as.character, # VARBINARY |
|||
"-2" = as.character, # BINARY |
|||
"-1" = as.character, # LONGVARCHAR |
|||
"0" = as.character, # NULL |
|||
"1" = as.character, # CHAR |
|||
"2" = as.double, # NUMERIC |
|||
"3" = as.double, # DECIMAL |
|||
"4" = as.integer, # INTEGER |
|||
"5" = as.integer, # SMALLINT |
|||
"6" = as.double, # FLOAT |
|||
"7" = as.double, # REAL |
|||
"8" = as.double, # DOUBLE |
|||
"12" = as.character, # VARCHAR |
|||
"16" = as_logical, # BOOLEAN |
|||
"91" = as_date, # DATE |
|||
"92" = as.character, # TIME |
|||
"93" = as_posixct, # TIMESTAMP |
|||
"2003" = as.character, # ARRAY |
|||
"1111" = as.character # OTHER |
|||
) -> .jdbc_converters |
|||
|
|||
#' @export |
|||
#' @keywords internal |
|||
setMethod("dbGetInfo", "AthenaDriver", def=function(dbObj, ...) |
|||
list( |
|||
name = "AthenaJDBC", |
|||
driver_version = list.files(system.file("java", package="metis.lite"), "jar$")[1], |
|||
package_version = utils::packageVersion("metis.lite") |
|||
) |
|||
) |
|||
|
|||
#' @export |
|||
#' @keywords internal |
|||
setMethod("dbGetInfo", "AthenaConnection", def=function(dbObj, ...) |
|||
list( |
|||
name = "AthenaJDBC", |
|||
driver_version = list.files(system.file("java", package="metis.lite"), "jar$")[1], |
|||
package_version = utils::packageVersion("metis.lite") |
|||
) |
|||
) |
|||
|
|||
|
|||
#' Fetch records from a previously executed query |
|||
#' |
|||
#' Fetch the next `n` elements (rows) from the result set and return them |
|||
#' as a data.frame. |
|||
#' |
|||
#' @param res An object inheriting from [DBIResult-class], created by |
|||
#' [dbSendQuery()]. |
|||
#' @param n maximum number of records to retrieve per fetch. Use `n = -1` |
|||
#' or `n = Inf` |
|||
#' to retrieve all pending records. Some implementations may recognize other |
|||
#' special values. |
|||
#' @param ... Other arguments passed on to methods. |
|||
#' @export |
|||
setMethod( |
|||
"fetch", |
|||
signature(res="AthenaResult", n="numeric"), |
|||
def = function(res, n, block = 1000L, ...) { |
|||
|
|||
nms <- c() |
|||
athena_type_convert <- list() |
|||
|
|||
cols <- .jcall(res@md, "I", "getColumnCount") |
|||
|
|||
for (i in 1:cols) { |
|||
ct <- as.character(.jcall(res@md, "I", "getColumnType", i)) |
|||
athena_type_convert[[i]] <- .jdbc_converters[[ct]] |
|||
nms <- c(nms, .jcall(res@md, "S", "getColumnLabel", i)) |
|||
# message(ct, "|", tail(nms, 1)) |
|||
} |
|||
|
|||
athena_type_convert <- set_names(athena_type_convert, nms) |
|||
|
|||
out <- callNextMethod(res = res, n = n, block = block, ...) |
|||
|
|||
# print(str(out)) |
|||
|
|||
for (nm in names(athena_type_convert)) { |
|||
f <- athena_type_convert[[nm]] |
|||
if (length(f) == 0) f <- as.character # catchall in case AMZN is tricksy |
|||
out[[nm]] <- f(out[[nm]]) |
|||
} |
|||
|
|||
out |
|||
|
|||
} |
|||
) |
|||
|
|||
#' AthenaJDBC |
|||
#' |
|||
#' @param conn Athena connection |
|||
#' @param statement SQL statement |
|||
#' @param ... unused |
|||
#' @importFrom rJava .jcall |
|||
#' @export |
|||
setMethod( |
|||
|
|||
"dbGetQuery", |
|||
signature(conn="AthenaConnection", statement="character"), |
|||
|
|||
definition = function(conn, statement, ...) { |
|||
|
|||
r <- dbSendQuery(conn, statement, ...) |
|||
|
|||
on.exit(.jcall(r@stat, "V", "close")) |
|||
|
|||
res <- fetch(r, -1, block = conn@fetch_size) |
|||
|
|||
class(res) <- c("tbl_df", "tbl", "data.frame") |
|||
|
|||
res |
|||
|
|||
} |
|||
|
|||
) |
|||
@ -1,11 +1,2 @@ |
|||
.onLoad <- function(libname, pkgname) { |
|||
rJava::.jpackage(pkgname, jars = "*", lib.loc = libname) |
|||
rJava::.jaddClassPath(dir(file.path(getwd(), "inst/java"), full.names = TRUE)) |
|||
o <- getOption("java.parameters", "") |
|||
if (!any(grepl("-Xrs", o))) { |
|||
packageStartupMessage( |
|||
"Did not find '-Xrs' in java.parameters option. Until rJava is updated, ", |
|||
"please set this up in your/an Rprofile or at the start of scripts." |
|||
) |
|||
} |
|||
} |
|||
|
|||
@ -1,18 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r, R/sql_translate_env.R |
|||
\name{Athena} |
|||
\alias{Athena} |
|||
\alias{db_data_type.AthenaConnection} |
|||
\alias{sql_translate_env.AthenaConnection} |
|||
\title{AthenaJDBC} |
|||
\usage{ |
|||
Athena(identifier.quote = "`") |
|||
|
|||
db_data_type.AthenaConnection(con, fields, ...) |
|||
|
|||
sql_translate_env.AthenaConnection(con) |
|||
} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
\keyword{internal} |
|||
@ -1,9 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{class} |
|||
\name{AthenaConnection-class} |
|||
\alias{AthenaConnection-class} |
|||
\title{AthenaJDBC} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -1,9 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{class} |
|||
\name{AthenaDriver-class} |
|||
\alias{AthenaDriver-class} |
|||
\title{AthenaJDBC} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -1,9 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{class} |
|||
\name{AthenaResult-class} |
|||
\alias{AthenaResult-class} |
|||
\title{AthenaJDBC} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -1,57 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/metis.r |
|||
\name{athena_connect} |
|||
\alias{athena_connect} |
|||
\title{Simplified Athena JDBC connection helper} |
|||
\usage{ |
|||
athena_connect(default_schema = "default", region = c("us-east-1", |
|||
"us-east-2", "us-west-2"), |
|||
s3_staging_dir = Sys.getenv("AWS_S3_STAGING_DIR"), |
|||
max_error_retries = 10, connection_timeout = 10000, |
|||
socket_timeout = 10000, log_path = "", log_level = c("OFF", |
|||
"FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE"), ...) |
|||
} |
|||
\arguments{ |
|||
\item{default_schema}{default schema (you'll still need to fully qualify non-default schema table names)} |
|||
|
|||
\item{region}{AWS region (Ref: \url{http://docs.aws.amazon.com/general/latest/gr/rande.html#athena})} |
|||
|
|||
\item{s3_staging_dir}{the Amazon S3 location to which your query output is written. The JDBC driver then asks Athena to read the results and provide rows of data back to the user.} |
|||
|
|||
\item{max_error_retries}{the maximum number of retries that the JDBC client attempts to make a request to Athena.} |
|||
|
|||
\item{connection_timeout}{the maximum amount of time, in milliseconds, to make a successful connection to Athena before an attempt is terminated.} |
|||
|
|||
\item{socket_timeout}{the maximum amount of time, in milliseconds, to wait for a socket in order to send data to Athena.} |
|||
|
|||
\item{log_path}{local path of the Athena JDBC driver logs. If no log path is provided, then no log files are created.} |
|||
|
|||
\item{log_level}{log level of the Athena JDBC driver logs. Use names |
|||
"OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE".} |
|||
|
|||
\item{...}{passed on to the driver} |
|||
} |
|||
\description{ |
|||
Handles the up-front JDBC config |
|||
} |
|||
\examples{ |
|||
\dontrun{ |
|||
use_credentials("personal") |
|||
|
|||
athena_connect( |
|||
default_schema = "sampledb", |
|||
s3_staging_dir = "s3://accessible-bucket", |
|||
log_path = "/tmp/athena.log", |
|||
log_level = "DEBUG" |
|||
) -> ath |
|||
|
|||
dbListTables(ath) |
|||
|
|||
dbGetQuery(ath, "SELECT * FROM sampledb.elb_logs LIMIT 1") |
|||
|
|||
} |
|||
} |
|||
\references{ |
|||
\href{https://docs.aws.amazon.com/athena/latest/ug/connect-with-jdbc.html}{Connect with JDBC}; |
|||
\href{https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf}{Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide} |
|||
} |
|||
@ -1,48 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{methods} |
|||
\name{dbConnect,AthenaDriver-method} |
|||
\alias{dbConnect,AthenaDriver-method} |
|||
\title{AthenaJDBC} |
|||
\usage{ |
|||
\S4method{dbConnect}{AthenaDriver}(drv, |
|||
provider = "com.simba.athena.amazonaws.auth.DefaultAWSCredentialsProviderChain", |
|||
region = "us-east-1", |
|||
s3_staging_dir = Sys.getenv("AWS_S3_STAGING_DIR"), |
|||
schema_name = "default", fetch_size = 1000L, |
|||
max_error_retries = 10, connection_timeout = 10000, |
|||
socket_timeout = 10000, log_path = "", log_level = 0, ...) |
|||
} |
|||
\arguments{ |
|||
\item{provider}{JDBC auth provider (ideally leave default)} |
|||
|
|||
\item{region}{AWS region the Athena tables are in} |
|||
|
|||
\item{s3_staging_dir}{A write-able bucket on S3 that you have permissions for} |
|||
|
|||
\item{schema_name}{LOL if only this actually worked with Amazon's hacked Presto driver} |
|||
|
|||
\item{max_error_retries, connection_timeout, socket_timeout}{technical connection info that you should only muck with if you know what you're doing.} |
|||
|
|||
\item{log_path, log_level}{The Athena JDBC driver can (shockingly) provide a decent bit |
|||
of data in logs. Set this to a temporary directory or something log4j can use. For |
|||
`log_level` use the names ("INFO", "DEBUG", "WARN", "ERROR", "ALL", "OFF", "FATAL", "TRACE") or |
|||
their corresponding integer values 0-6.} |
|||
|
|||
\item{...}{passed on to the driver. See Details.} |
|||
} |
|||
\description{ |
|||
Connect to Athena |
|||
} |
|||
\section{Driver Configuration Options}{ |
|||
|
|||
|
|||
- `BinaryColumnLength`: <int> The maximum data length for `BINARY` columns. Default `32767L` |
|||
- `ComplexTypeColumnLength`: <int> The maximum data length for `ARRAY`, `MAP`, and `STRUCT` columns. Default `65535L` |
|||
- `StringColumnLength`: <int> The maximum data length for `STRING` columns. Default `255L` |
|||
} |
|||
|
|||
\references{ |
|||
[Connect with JDBC](https://docs.aws.amazon.com/athena/latest/ug/connect-with-jdbc.html); |
|||
[Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf) |
|||
} |
|||
@ -1,22 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{methods} |
|||
\name{dbExistsTable,AthenaConnection,character-method} |
|||
\alias{dbExistsTable,AthenaConnection,character-method} |
|||
\title{AthenaJDBC} |
|||
\usage{ |
|||
\S4method{dbExistsTable}{AthenaConnection,character}(conn, name, schema, |
|||
...) |
|||
} |
|||
\arguments{ |
|||
\item{conn}{Athena connection} |
|||
|
|||
\item{name}{table name} |
|||
|
|||
\item{schema}{Athena schema name} |
|||
|
|||
\item{...}{unused} |
|||
} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -1,19 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/z-dbGetQuery.R |
|||
\docType{methods} |
|||
\name{dbGetQuery,AthenaConnection,character-method} |
|||
\alias{dbGetQuery,AthenaConnection,character-method} |
|||
\title{AthenaJDBC} |
|||
\usage{ |
|||
\S4method{dbGetQuery}{AthenaConnection,character}(conn, statement, ...) |
|||
} |
|||
\arguments{ |
|||
\item{conn}{Athena connection} |
|||
|
|||
\item{statement}{SQL statement} |
|||
|
|||
\item{...}{unused} |
|||
} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -1,22 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{methods} |
|||
\name{dbListFields,AthenaConnection,character-method} |
|||
\alias{dbListFields,AthenaConnection,character-method} |
|||
\title{AthenaJDBC} |
|||
\usage{ |
|||
\S4method{dbListFields}{AthenaConnection,character}(conn, name, schema, |
|||
...) |
|||
} |
|||
\arguments{ |
|||
\item{conn}{Athena connection} |
|||
|
|||
\item{name}{table name} |
|||
|
|||
\item{schema}{Athena schema name} |
|||
|
|||
\item{...}{unused} |
|||
} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -1,22 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{methods} |
|||
\name{dbListTables,AthenaConnection-method} |
|||
\alias{dbListTables,AthenaConnection-method} |
|||
\title{AthenaJDBC} |
|||
\usage{ |
|||
\S4method{dbListTables}{AthenaConnection}(conn, pattern = "*", schema, |
|||
...) |
|||
} |
|||
\arguments{ |
|||
\item{conn}{Athena connection} |
|||
|
|||
\item{pattern}{table name pattern} |
|||
|
|||
\item{schema}{Athena schema name} |
|||
|
|||
\item{...}{unused} |
|||
} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -1,21 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{methods} |
|||
\name{dbReadTable,AthenaConnection,character-method} |
|||
\alias{dbReadTable,AthenaConnection,character-method} |
|||
\title{AthenaJDBC} |
|||
\usage{ |
|||
\S4method{dbReadTable}{AthenaConnection,character}(conn, name, schema, ...) |
|||
} |
|||
\arguments{ |
|||
\item{conn}{Athena connection} |
|||
|
|||
\item{name}{table name} |
|||
|
|||
\item{schema}{Athena schema name} |
|||
|
|||
\item{...}{unused} |
|||
} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -1,19 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/jdbc.r |
|||
\docType{methods} |
|||
\name{dbSendQuery,AthenaConnection,character-method} |
|||
\alias{dbSendQuery,AthenaConnection,character-method} |
|||
\title{AthenaJDBC} |
|||
\usage{ |
|||
\S4method{dbSendQuery}{AthenaConnection,character}(conn, statement, ...) |
|||
} |
|||
\arguments{ |
|||
\item{conn}{Athena connection} |
|||
|
|||
\item{statement}{SQL statement} |
|||
|
|||
\item{...}{unused} |
|||
} |
|||
\description{ |
|||
AthenaJDBC |
|||
} |
|||
@ -0,0 +1,25 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/sql_translate_env.R |
|||
\name{db_data_type.AthenaConnection} |
|||
\alias{db_data_type.AthenaConnection} |
|||
\alias{sql_translate_env.AthenaConnection} |
|||
\title{Convert R data type to Athena} |
|||
\usage{ |
|||
db_data_type.AthenaConnection(con, fields, ...) |
|||
|
|||
sql_translate_env.AthenaConnection(con) |
|||
} |
|||
\arguments{ |
|||
\item{con}{Athena connection} |
|||
|
|||
\item{fields}{fields to type reference} |
|||
|
|||
\item{...}{ignored} |
|||
|
|||
\item{con}{Athena connection} |
|||
} |
|||
\description{ |
|||
Convert R data type to Athena |
|||
|
|||
Translate R tridyverse functional idioms to Athena |
|||
} |
|||
@ -1,24 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/z-dbGetQuery.R |
|||
\docType{methods} |
|||
\name{fetch,AthenaResult,numeric-method} |
|||
\alias{fetch,AthenaResult,numeric-method} |
|||
\title{Fetch records from a previously executed query} |
|||
\usage{ |
|||
\S4method{fetch}{AthenaResult,numeric}(res, n, block = 1000L, ...) |
|||
} |
|||
\arguments{ |
|||
\item{res}{An object inheriting from [DBIResult-class], created by |
|||
[dbSendQuery()].} |
|||
|
|||
\item{n}{maximum number of records to retrieve per fetch. Use `n = -1` |
|||
or `n = Inf` |
|||
to retrieve all pending records. Some implementations may recognize other |
|||
special values.} |
|||
|
|||
\item{...}{Other arguments passed on to methods.} |
|||
} |
|||
\description{ |
|||
Fetch the next `n` elements (rows) from the result set and return them |
|||
as a data.frame. |
|||
} |
|||
@ -1,35 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/metis-lite-package.R |
|||
\docType{package} |
|||
\encoding{UTF-8} |
|||
\name{metis.lite} |
|||
\alias{metis.lite} |
|||
\alias{metis.lite-package} |
|||
\title{Access and Query Amazon Athena via DBI/JDBC} |
|||
\description{ |
|||
Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, |
|||
perform queries and retrieve query results. A lightweight 'RJDBC' implementation |
|||
is included along with an interface to the 'AWS' command-line utility. |
|||
} |
|||
\section{IMPORTANT}{ |
|||
|
|||
|
|||
Since R 3.5 (I don't remember this happening in R 3.4.x) signals sent from interrupting |
|||
Athena JDBC calls crash the R #' interpreter. You need to set the \code{-Xrs} option to avoid |
|||
signals being passed on to the JVM owner. That has to be done \emph{before} \code{rJava} is |
|||
loaded so you either need to remember to put it at the top of all scripts \emph{or} stick this |
|||
in your local #' \code{~/.Rprofile} and/or sitewide \code{Rprofile}:\preformatted{if (!grepl("-Xrs", getOption("java.parameters", ""))) { |
|||
options( |
|||
"java.parameters" = c(getOption("java.parameters", default = NULL), "-Xrs") |
|||
) |
|||
} |
|||
} |
|||
} |
|||
|
|||
\references{ |
|||
\href{https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf}{Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide} |
|||
} |
|||
\author{ |
|||
Bob Rudis (bob@rud.is) |
|||
} |
|||
\keyword{internal} |
|||
@ -0,0 +1,19 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/metis-tidy-package.R |
|||
\docType{package} |
|||
\encoding{UTF-8} |
|||
\name{metis.tidy} |
|||
\alias{metis.tidy} |
|||
\alias{metis.tidy-package} |
|||
\title{Access and Query Amazon Athena via the Tidyverse} |
|||
\description{ |
|||
Methods are provided to use the 'metis' JDBC/DBI interface via |
|||
the Tidyverse (e.g. 'dbplyr'/'dplyr' idioms). |
|||
} |
|||
\references{ |
|||
\href{https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf}{Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide} |
|||
} |
|||
\author{ |
|||
Bob Rudis (bob@rud.is) |
|||
} |
|||
\keyword{internal} |
|||
@ -1,12 +0,0 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/metis-lite-package.R |
|||
\name{use_credentials} |
|||
\alias{use_credentials} |
|||
\alias{read_credentials} |
|||
\title{Use Credentials from .aws/credentials File} |
|||
\description{ |
|||
Use Credentials from .aws/credentials File |
|||
} |
|||
\references{ |
|||
\code{\link[aws.signature:use_credentials]{aws.signature::use_credentials()}} / \code{\link[aws.signature:read_credentials]{aws.signature::read_credentials()}} |
|||
} |
|||
@ -1,2 +1,2 @@ |
|||
library(testthat) |
|||
test_check("metis-lite") |
|||
test_check("metis.tidy") |
|||
|
|||
Loading…
Reference in new issue