diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 69cb760..0000000 --- a/.codecov.yml +++ /dev/null @@ -1 +0,0 @@ -comment: false diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 76d9586..0000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: r - -warnings_are_errors: true - -sudo: required - -cache: packages - -r: - - oldrel - - release - - devel - -apt_packages: - - libv8-dev - - xclip - -env: - global: - - CRAN: http://cran.rstudio.com - -after_success: - - Rscript -e 'covr::codecov()' - -notifications: - email: - - bob@rud.is - irc: - channels: - - "104.236.112.222#builds" - nick: travisci diff --git a/DESCRIPTION b/DESCRIPTION index 74ca400..977e571 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,21 +1,17 @@ -Package: metis +Package: metis.lite Type: Package Title: Helpers for Accessing and Querying Amazon Athena -Version: 0.4.0 -Date: 2018-03-19 +Version: 0.3.0 +Date: 2019-02-14 Authors@R: c( person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"), - comment = c(ORCID = "0000-0001-5670-2640")), - person("Derek", "Abdine", comment = "Authentication driver update"), - person("Zachary", "Kurtz", email = "zdkurtz@gmail.com", role = "ctb") + comment = c(ORCID = "0000-0001-5670-2640")) ) Maintainer: Bob Rudis Encoding: UTF-8 Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, perform queries and retrieve query results. A lightweight 'RJDBC' implementation is included along with an interface to the 'AWS' command-line utility. -URL: https://github.com/hrbrmstr/metis -BugReports: https://github.com/hrbrmstr/metis/issues SystemRequirements: JDK 1.8+ License: AGPL Suggests: @@ -31,6 +27,5 @@ Imports: readr, aws.signature, uuid, - reticulate, jsonlite RoxygenNote: 6.1.1 diff --git a/NAMESPACE b/NAMESPACE index 1f7afe0..c14e274 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,6 @@ # Generated by roxygen2: do not edit by hand +S3method(db_data_type,AthenaConnection) S3method(sql_translate_env,AthenaConnection) export(Athena) export(athena_connect) diff --git a/NEWS.md b/NEWS.md index 789cc4f..12a2e7d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,8 @@ -0.2.0 +0.3.0 ========= - Added `sql_translate_env.AthenaConnection()` +- Handles Athena JDBC column types! 0.2.0 ========= diff --git a/R/jdbc.r b/R/jdbc.r index 1362d21..8c97224 100644 --- a/R/jdbc.r +++ b/R/jdbc.r @@ -1,3 +1,28 @@ +.jt <- list( + BIT = -7, + TINYINT = -6, + BIGINT = -5, + LONGVARBINARY = -4, + VARBINARY = -3, + BINARY = -2, + LONGVARCHAR = -1, + .NULL = 0, + CHAR = 1, + NUMERIC = 2, + DECIMAL = 3, + INTEGER = 4, + SMALLINT = 5, + FLOAT = 6, + REAL = 7, + DOUBLE = 8, + VARCHAR = 12, + DATE = 91, + TIME = 92, + TIMESTAMP = 93, + OTHER = 1111 +) + + stats::setNames( 0:6, c("OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE") @@ -137,11 +162,70 @@ setMethod( signature(conn="AthenaConnection", statement="character"), definition = function(conn, statement, type_convert=FALSE, ...) { + r <- dbSendQuery(conn, statement, ...) + on.exit(.jcall(r@stat, "V", "close")) - res <- dplyr::tbl_df(fetch(r, -1, block=1000)) - if (type_convert) res <- readr::type_convert(res) + + structure( + list( + cols = list(), + default = structure(list(), class = c("collector_guess", "collector")) + ), + class = "col_spec" + ) -> l + + cols <- .jcall(r@md, "I", "getColumnCount") + nms <- c() + + cts <- rep(0L, cols) + for (i in 1:cols) { + ct <- .jcall(r@md, "I", "getColumnType", i) + if (ct == .jt$CHAR) { + l$cols[[i]] <- col_character() + } else if (ct == .jt$NUMERIC) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$DECIMAL) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$INTEGER) { + l$cols[[i]] <- col_integer() + } else if (ct == .jt$SMALLINT) { + l$cols[[i]] <- col_integer() + } else if (ct == .jt$TINYINT) { + l$cols[[i]] <- col_integer() + } else if (ct == .jt$BIGINT) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$FLOAT) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$REAL) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$DOUBLE) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$VARCHAR) { + l$cols[[i]] <- col_character() + } else if (ct == .jt$DATE) { + l$cols[[i]] <- col_date() + } else if (ct == .jt$TIME) { + l$cols[[i]] <- col_time() + } else if (ct == .jt$TIMESTAMP) { + l$cols[[i]] <- col_datetime() + } else if (ct == .jt$BIT) { + l$cols[[i]] <- col_logical() + } else { + l$cols[[i]] <- col_character() + } + nms[i] <- .jcall(r@md, "S", "getColumnLabel", i) + } + + l$cols <- stats::setNames(l$cols, nms) + + res <- fetch(r, -1, block=1000) + res <- readr::type_convert(res, col_types = l) + + class(res) <- c("tbl_df", "tbl", "data.frame") + res + } ) diff --git a/R/metis-package.R b/R/metis-lite-package.R similarity index 97% rename from R/metis-package.R rename to R/metis-lite-package.R index af572b1..5ecd549 100644 --- a/R/metis-package.R +++ b/R/metis-lite-package.R @@ -4,7 +4,7 @@ #' perform queries and retrieve query results. A lightweight 'RJDBC' implementation #' is included along with an interface to the 'AWS' command-line utility. #' -#' @name metis +#' @name metis.lite #' @encoding UTF-8 #' @docType package #' @author Bob Rudis (bob@@rud.is) diff --git a/R/sql_translate_env.R b/R/sql_translate_env.R index 0de8bef..4467278 100644 --- a/R/sql_translate_env.R +++ b/R/sql_translate_env.R @@ -1,6 +1,29 @@ #' @rdname Athena #' @keywords internal #' @export +db_data_type.AthenaConnection <- function(con, fields, ...) { + print("\n\n\ndb_data_type\n\n\n") + data_type <- function(x) { + switch( + class(x)[1], + integer64 = "BIGINT", + logical = "BOOLEAN", + integer = "INTEGER", + numeric = "DOUBLE", + factor = "CHARACTER", + character = "CHARACTER", + Date = "DATE", + POSIXct = "TIMESTAMP", + stop("Can't map type ", paste(class(x), collapse = "/"), + " to a supported database type.") + ) + } + vapply(fields, data_type, character(1)) +} + +#' @rdname Athena +#' @keywords internal +#' @export sql_translate_env.AthenaConnection <- function(con) { x <- con diff --git a/R/zzz.R b/R/zzz.R index b20e5a3..588f6a7 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,4 +1,4 @@ .onLoad <- function(libname, pkgname) { - # rJava::.jpackage(pkgname, jars = "*", lib.loc = libname) - rJava::.jaddClassPath(dir(file.path(getwd(), "inst/java"), full.names = TRUE)) + rJava::.jpackage(pkgname, jars = "*", lib.loc = libname) + # rJava::.jaddClassPath(dir(file.path(getwd(), "inst/java"), full.names = TRUE)) } diff --git a/man/Athena.Rd b/man/Athena.Rd index dea61e7..0f5dfa5 100644 --- a/man/Athena.Rd +++ b/man/Athena.Rd @@ -2,11 +2,14 @@ % Please edit documentation in R/jdbc.r, R/sql_translate_env.R \name{Athena} \alias{Athena} +\alias{db_data_type.AthenaConnection} \alias{sql_translate_env.AthenaConnection} \title{AthenaJDBC} \usage{ Athena(identifier.quote = "`") +\method{db_data_type}{AthenaConnection}(con, fields, ...) + \method{sql_translate_env}{AthenaConnection}(con) } \description{ diff --git a/man/metis.Rd b/man/metis.lite.Rd similarity index 77% rename from man/metis.Rd rename to man/metis.lite.Rd index a91bba3..6651d47 100644 --- a/man/metis.Rd +++ b/man/metis.lite.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/metis-package.R +% Please edit documentation in R/metis-lite-package.R \docType{package} \encoding{UTF-8} -\name{metis} -\alias{metis} -\alias{metis-package} +\name{metis.lite} +\alias{metis.lite} +\alias{metis.lite-package} \title{Helpers for Accessing and Querying Amazon Athena} \description{ Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, diff --git a/man/use_credentials.Rd b/man/use_credentials.Rd index 8d9fe15..b6a2060 100644 --- a/man/use_credentials.Rd +++ b/man/use_credentials.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/metis-package.R +% Please edit documentation in R/metis-lite-package.R \name{use_credentials} \alias{use_credentials} \alias{read_credentials} diff --git a/metis.Rproj b/metis-lite.Rproj similarity index 100% rename from metis.Rproj rename to metis-lite.Rproj diff --git a/tests/test-all.R b/tests/test-all.R index 9bebc31..fb8feef 100644 --- a/tests/test-all.R +++ b/tests/test-all.R @@ -1,2 +1,2 @@ library(testthat) -test_check("metis") +test_check("metis-lite") diff --git a/tests/testthat/test-metis.R b/tests/testthat/test-metis-lite.R similarity index 100% rename from tests/testthat/test-metis.R rename to tests/testthat/test-metis-lite.R