From 041c62da0c8589ced227b56d7a16ede28de025f2 Mon Sep 17 00:00:00 2001 From: hrbrmstr Date: Fri, 15 Feb 2019 17:45:28 -0500 Subject: [PATCH] handles types\! --- .codecov.yml | 1 - .travis.yml | 31 -------------- DESCRIPTION | 13 ++---- NAMESPACE | 1 + NEWS.md | 3 +- R/jdbc.r | 88 +++++++++++++++++++++++++++++++++++++++- R/metis-lite-package.R | 35 ++++++++++++++++ R/metis-package.R | 35 ---------------- R/sql_translate_env.R | 23 +++++++++++ R/zzz.R | 4 +- man/Athena.Rd | 3 ++ man/metis.Rd | 16 -------- man/metis.lite.Rd | 16 ++++++++ man/use_credentials.Rd | 2 +- metis-lite.Rproj | 21 ++++++++++ metis.Rproj | 21 ---------- tests/test-all.R | 2 +- tests/testthat/test-metis-lite.R | 6 +++ tests/testthat/test-metis.R | 6 --- 19 files changed, 201 insertions(+), 126 deletions(-) delete mode 100644 .codecov.yml delete mode 100644 .travis.yml create mode 100644 R/metis-lite-package.R delete mode 100644 R/metis-package.R delete mode 100644 man/metis.Rd create mode 100644 man/metis.lite.Rd create mode 100644 metis-lite.Rproj delete mode 100644 metis.Rproj create mode 100644 tests/testthat/test-metis-lite.R delete mode 100644 tests/testthat/test-metis.R diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 69cb760..0000000 --- a/.codecov.yml +++ /dev/null @@ -1 +0,0 @@ -comment: false diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 76d9586..0000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: r - -warnings_are_errors: true - -sudo: required - -cache: packages - -r: - - oldrel - - release - - devel - -apt_packages: - - libv8-dev - - xclip - -env: - global: - - CRAN: http://cran.rstudio.com - -after_success: - - Rscript -e 'covr::codecov()' - -notifications: - email: - - bob@rud.is - irc: - channels: - - "104.236.112.222#builds" - nick: travisci diff --git a/DESCRIPTION b/DESCRIPTION index 74ca400..977e571 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,21 +1,17 @@ -Package: metis +Package: metis.lite Type: Package Title: Helpers for Accessing and Querying Amazon Athena -Version: 0.4.0 -Date: 2018-03-19 +Version: 0.3.0 +Date: 2019-02-14 Authors@R: c( person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"), - comment = c(ORCID = "0000-0001-5670-2640")), - person("Derek", "Abdine", comment = "Authentication driver update"), - person("Zachary", "Kurtz", email = "zdkurtz@gmail.com", role = "ctb") + comment = c(ORCID = "0000-0001-5670-2640")) ) Maintainer: Bob Rudis Encoding: UTF-8 Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, perform queries and retrieve query results. A lightweight 'RJDBC' implementation is included along with an interface to the 'AWS' command-line utility. -URL: https://github.com/hrbrmstr/metis -BugReports: https://github.com/hrbrmstr/metis/issues SystemRequirements: JDK 1.8+ License: AGPL Suggests: @@ -31,6 +27,5 @@ Imports: readr, aws.signature, uuid, - reticulate, jsonlite RoxygenNote: 6.1.1 diff --git a/NAMESPACE b/NAMESPACE index 1f7afe0..c14e274 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,6 @@ # Generated by roxygen2: do not edit by hand +S3method(db_data_type,AthenaConnection) S3method(sql_translate_env,AthenaConnection) export(Athena) export(athena_connect) diff --git a/NEWS.md b/NEWS.md index 789cc4f..12a2e7d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,8 @@ -0.2.0 +0.3.0 ========= - Added `sql_translate_env.AthenaConnection()` +- Handles Athena JDBC column types! 0.2.0 ========= diff --git a/R/jdbc.r b/R/jdbc.r index 1362d21..8c97224 100644 --- a/R/jdbc.r +++ b/R/jdbc.r @@ -1,3 +1,28 @@ +.jt <- list( + BIT = -7, + TINYINT = -6, + BIGINT = -5, + LONGVARBINARY = -4, + VARBINARY = -3, + BINARY = -2, + LONGVARCHAR = -1, + .NULL = 0, + CHAR = 1, + NUMERIC = 2, + DECIMAL = 3, + INTEGER = 4, + SMALLINT = 5, + FLOAT = 6, + REAL = 7, + DOUBLE = 8, + VARCHAR = 12, + DATE = 91, + TIME = 92, + TIMESTAMP = 93, + OTHER = 1111 +) + + stats::setNames( 0:6, c("OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE") @@ -137,11 +162,70 @@ setMethod( signature(conn="AthenaConnection", statement="character"), definition = function(conn, statement, type_convert=FALSE, ...) { + r <- dbSendQuery(conn, statement, ...) + on.exit(.jcall(r@stat, "V", "close")) - res <- dplyr::tbl_df(fetch(r, -1, block=1000)) - if (type_convert) res <- readr::type_convert(res) + + structure( + list( + cols = list(), + default = structure(list(), class = c("collector_guess", "collector")) + ), + class = "col_spec" + ) -> l + + cols <- .jcall(r@md, "I", "getColumnCount") + nms <- c() + + cts <- rep(0L, cols) + for (i in 1:cols) { + ct <- .jcall(r@md, "I", "getColumnType", i) + if (ct == .jt$CHAR) { + l$cols[[i]] <- col_character() + } else if (ct == .jt$NUMERIC) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$DECIMAL) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$INTEGER) { + l$cols[[i]] <- col_integer() + } else if (ct == .jt$SMALLINT) { + l$cols[[i]] <- col_integer() + } else if (ct == .jt$TINYINT) { + l$cols[[i]] <- col_integer() + } else if (ct == .jt$BIGINT) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$FLOAT) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$REAL) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$DOUBLE) { + l$cols[[i]] <- col_double() + } else if (ct == .jt$VARCHAR) { + l$cols[[i]] <- col_character() + } else if (ct == .jt$DATE) { + l$cols[[i]] <- col_date() + } else if (ct == .jt$TIME) { + l$cols[[i]] <- col_time() + } else if (ct == .jt$TIMESTAMP) { + l$cols[[i]] <- col_datetime() + } else if (ct == .jt$BIT) { + l$cols[[i]] <- col_logical() + } else { + l$cols[[i]] <- col_character() + } + nms[i] <- .jcall(r@md, "S", "getColumnLabel", i) + } + + l$cols <- stats::setNames(l$cols, nms) + + res <- fetch(r, -1, block=1000) + res <- readr::type_convert(res, col_types = l) + + class(res) <- c("tbl_df", "tbl", "data.frame") + res + } ) diff --git a/R/metis-lite-package.R b/R/metis-lite-package.R new file mode 100644 index 0000000..5ecd549 --- /dev/null +++ b/R/metis-lite-package.R @@ -0,0 +1,35 @@ +#' Helpers for Accessing and Querying Amazon Athena +#' +#' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, +#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation +#' is included along with an interface to the 'AWS' command-line utility. +#' +#' @name metis.lite +#' @encoding UTF-8 +#' @docType package +#' @author Bob Rudis (bob@@rud.is) +#' @import RJDBC +#' @import DBI +#' @import dplyr +#' @importFrom jsonlite fromJSON +#' @importFrom readr type_convert +#' @importFrom uuid UUIDgenerate +#' @importFrom sys exec_internal +#' @importFrom aws.signature use_credentials read_credentials +NULL + + +#' Use Credentials from .aws/credentials File +#' +#' @md +#' @references [aws.signature::use_credentials()] / [aws.signature::read_credentials()] +#' @name use_credentials +#' @rdname use_credentials +#' @inheritParams aws.signature::use_credentials +#' @export +NULL + +#' @name read_credentials +#' @rdname use_credentials +#' @export +NULL diff --git a/R/metis-package.R b/R/metis-package.R deleted file mode 100644 index af572b1..0000000 --- a/R/metis-package.R +++ /dev/null @@ -1,35 +0,0 @@ -#' Helpers for Accessing and Querying Amazon Athena -#' -#' Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, -#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation -#' is included along with an interface to the 'AWS' command-line utility. -#' -#' @name metis -#' @encoding UTF-8 -#' @docType package -#' @author Bob Rudis (bob@@rud.is) -#' @import RJDBC -#' @import DBI -#' @import dplyr -#' @importFrom jsonlite fromJSON -#' @importFrom readr type_convert -#' @importFrom uuid UUIDgenerate -#' @importFrom sys exec_internal -#' @importFrom aws.signature use_credentials read_credentials -NULL - - -#' Use Credentials from .aws/credentials File -#' -#' @md -#' @references [aws.signature::use_credentials()] / [aws.signature::read_credentials()] -#' @name use_credentials -#' @rdname use_credentials -#' @inheritParams aws.signature::use_credentials -#' @export -NULL - -#' @name read_credentials -#' @rdname use_credentials -#' @export -NULL diff --git a/R/sql_translate_env.R b/R/sql_translate_env.R index 0de8bef..4467278 100644 --- a/R/sql_translate_env.R +++ b/R/sql_translate_env.R @@ -1,6 +1,29 @@ #' @rdname Athena #' @keywords internal #' @export +db_data_type.AthenaConnection <- function(con, fields, ...) { + print("\n\n\ndb_data_type\n\n\n") + data_type <- function(x) { + switch( + class(x)[1], + integer64 = "BIGINT", + logical = "BOOLEAN", + integer = "INTEGER", + numeric = "DOUBLE", + factor = "CHARACTER", + character = "CHARACTER", + Date = "DATE", + POSIXct = "TIMESTAMP", + stop("Can't map type ", paste(class(x), collapse = "/"), + " to a supported database type.") + ) + } + vapply(fields, data_type, character(1)) +} + +#' @rdname Athena +#' @keywords internal +#' @export sql_translate_env.AthenaConnection <- function(con) { x <- con diff --git a/R/zzz.R b/R/zzz.R index b20e5a3..588f6a7 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,4 +1,4 @@ .onLoad <- function(libname, pkgname) { - # rJava::.jpackage(pkgname, jars = "*", lib.loc = libname) - rJava::.jaddClassPath(dir(file.path(getwd(), "inst/java"), full.names = TRUE)) + rJava::.jpackage(pkgname, jars = "*", lib.loc = libname) + # rJava::.jaddClassPath(dir(file.path(getwd(), "inst/java"), full.names = TRUE)) } diff --git a/man/Athena.Rd b/man/Athena.Rd index dea61e7..0f5dfa5 100644 --- a/man/Athena.Rd +++ b/man/Athena.Rd @@ -2,11 +2,14 @@ % Please edit documentation in R/jdbc.r, R/sql_translate_env.R \name{Athena} \alias{Athena} +\alias{db_data_type.AthenaConnection} \alias{sql_translate_env.AthenaConnection} \title{AthenaJDBC} \usage{ Athena(identifier.quote = "`") +\method{db_data_type}{AthenaConnection}(con, fields, ...) + \method{sql_translate_env}{AthenaConnection}(con) } \description{ diff --git a/man/metis.Rd b/man/metis.Rd deleted file mode 100644 index a91bba3..0000000 --- a/man/metis.Rd +++ /dev/null @@ -1,16 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/metis-package.R -\docType{package} -\encoding{UTF-8} -\name{metis} -\alias{metis} -\alias{metis-package} -\title{Helpers for Accessing and Querying Amazon Athena} -\description{ -Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, -perform queries and retrieve query results. A lightweight 'RJDBC' implementation -is included along with an interface to the 'AWS' command-line utility. -} -\author{ -Bob Rudis (bob@rud.is) -} diff --git a/man/metis.lite.Rd b/man/metis.lite.Rd new file mode 100644 index 0000000..6651d47 --- /dev/null +++ b/man/metis.lite.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/metis-lite-package.R +\docType{package} +\encoding{UTF-8} +\name{metis.lite} +\alias{metis.lite} +\alias{metis.lite-package} +\title{Helpers for Accessing and Querying Amazon Athena} +\description{ +Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables, +perform queries and retrieve query results. A lightweight 'RJDBC' implementation +is included along with an interface to the 'AWS' command-line utility. +} +\author{ +Bob Rudis (bob@rud.is) +} diff --git a/man/use_credentials.Rd b/man/use_credentials.Rd index 8d9fe15..b6a2060 100644 --- a/man/use_credentials.Rd +++ b/man/use_credentials.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/metis-package.R +% Please edit documentation in R/metis-lite-package.R \name{use_credentials} \alias{use_credentials} \alias{read_credentials} diff --git a/metis-lite.Rproj b/metis-lite.Rproj new file mode 100644 index 0000000..446d9e1 --- /dev/null +++ b/metis-lite.Rproj @@ -0,0 +1,21 @@ +Version: 1.0 + +RestoreWorkspace: Default +SaveWorkspace: Default +AlwaysSaveHistory: Default + +EnableCodeIndexing: Yes +UseSpacesForTab: Yes +NumSpacesForTab: 2 +Encoding: UTF-8 + +RnwWeave: Sweave +LaTeX: pdfLaTeX + +StripTrailingWhitespace: Yes + +BuildType: Package +PackageUseDevtools: Yes +PackageInstallArgs: --no-multiarch --with-keep.source +PackageBuildArgs: --resave-data +PackageRoxygenize: rd,collate,namespace diff --git a/metis.Rproj b/metis.Rproj deleted file mode 100644 index 446d9e1..0000000 --- a/metis.Rproj +++ /dev/null @@ -1,21 +0,0 @@ -Version: 1.0 - -RestoreWorkspace: Default -SaveWorkspace: Default -AlwaysSaveHistory: Default - -EnableCodeIndexing: Yes -UseSpacesForTab: Yes -NumSpacesForTab: 2 -Encoding: UTF-8 - -RnwWeave: Sweave -LaTeX: pdfLaTeX - -StripTrailingWhitespace: Yes - -BuildType: Package -PackageUseDevtools: Yes -PackageInstallArgs: --no-multiarch --with-keep.source -PackageBuildArgs: --resave-data -PackageRoxygenize: rd,collate,namespace diff --git a/tests/test-all.R b/tests/test-all.R index 9bebc31..fb8feef 100644 --- a/tests/test-all.R +++ b/tests/test-all.R @@ -1,2 +1,2 @@ library(testthat) -test_check("metis") +test_check("metis-lite") diff --git a/tests/testthat/test-metis-lite.R b/tests/testthat/test-metis-lite.R new file mode 100644 index 0000000..ab6f62f --- /dev/null +++ b/tests/testthat/test-metis-lite.R @@ -0,0 +1,6 @@ +context("basic functionality") +test_that("we can do something", { + + #expect_that(some_function(), is_a("data.frame")) + +}) diff --git a/tests/testthat/test-metis.R b/tests/testthat/test-metis.R deleted file mode 100644 index ab6f62f..0000000 --- a/tests/testthat/test-metis.R +++ /dev/null @@ -1,6 +0,0 @@ -context("basic functionality") -test_that("we can do something", { - - #expect_that(some_function(), is_a("data.frame")) - -})