Browse Source

handles types\!

master
boB Rudis 1 year ago
parent
commit
041c62da0c
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
15 changed files with 128 additions and 53 deletions
  1. +0
    -1
      .codecov.yml
  2. +0
    -31
      .travis.yml
  3. +4
    -9
      DESCRIPTION
  4. +1
    -0
      NAMESPACE
  5. +2
    -1
      NEWS.md
  6. +86
    -2
      R/jdbc.r
  7. +1
    -1
      R/metis-lite-package.R
  8. +23
    -0
      R/sql_translate_env.R
  9. +2
    -2
      R/zzz.R
  10. +3
    -0
      man/Athena.Rd
  11. +4
    -4
      man/metis.lite.Rd
  12. +1
    -1
      man/use_credentials.Rd
  13. +0
    -0
      metis-lite.Rproj
  14. +1
    -1
      tests/test-all.R
  15. +0
    -0
      tests/testthat/test-metis-lite.R

+ 0
- 1
.codecov.yml View File

@@ -1 +0,0 @@
comment: false

+ 0
- 31
.travis.yml View File

@@ -1,31 +0,0 @@
language: r

warnings_are_errors: true

sudo: required

cache: packages

r:
- oldrel
- release
- devel

apt_packages:
- libv8-dev
- xclip

env:
global:
- CRAN: http://cran.rstudio.com

after_success:
- Rscript -e 'covr::codecov()'

notifications:
email:
- bob@rud.is
irc:
channels:
- "104.236.112.222#builds"
nick: travisci

+ 4
- 9
DESCRIPTION View File

@@ -1,21 +1,17 @@
Package: metis
Package: metis.lite
Type: Package
Title: Helpers for Accessing and Querying Amazon Athena
Version: 0.4.0
Date: 2018-03-19
Version: 0.3.0
Date: 2019-02-14
Authors@R: c(
person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"),
comment = c(ORCID = "0000-0001-5670-2640")),
person("Derek", "Abdine", comment = "Authentication driver update"),
person("Zachary", "Kurtz", email = "zdkurtz@gmail.com", role = "ctb")
comment = c(ORCID = "0000-0001-5670-2640"))
)
Maintainer: Bob Rudis <bob@rud.is>
Encoding: UTF-8
Description: Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,
perform queries and retrieve query results. A lightweight 'RJDBC' implementation
is included along with an interface to the 'AWS' command-line utility.
URL: https://github.com/hrbrmstr/metis
BugReports: https://github.com/hrbrmstr/metis/issues
SystemRequirements: JDK 1.8+
License: AGPL
Suggests:
@@ -31,6 +27,5 @@ Imports:
readr,
aws.signature,
uuid,
reticulate,
jsonlite
RoxygenNote: 6.1.1

+ 1
- 0
NAMESPACE View File

@@ -1,5 +1,6 @@
# Generated by roxygen2: do not edit by hand

S3method(db_data_type,AthenaConnection)
S3method(sql_translate_env,AthenaConnection)
export(Athena)
export(athena_connect)


+ 2
- 1
NEWS.md View File

@@ -1,7 +1,8 @@
0.2.0
0.3.0
=========

- Added `sql_translate_env.AthenaConnection()`
- Handles Athena JDBC column types!

0.2.0
=========


+ 86
- 2
R/jdbc.r View File

@@ -1,3 +1,28 @@
.jt <- list(
BIT = -7,
TINYINT = -6,
BIGINT = -5,
LONGVARBINARY = -4,
VARBINARY = -3,
BINARY = -2,
LONGVARCHAR = -1,
.NULL = 0,
CHAR = 1,
NUMERIC = 2,
DECIMAL = 3,
INTEGER = 4,
SMALLINT = 5,
FLOAT = 6,
REAL = 7,
DOUBLE = 8,
VARCHAR = 12,
DATE = 91,
TIME = 92,
TIMESTAMP = 93,
OTHER = 1111
)


stats::setNames(
0:6,
c("OFF", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE")
@@ -137,11 +162,70 @@ setMethod(
signature(conn="AthenaConnection", statement="character"),

definition = function(conn, statement, type_convert=FALSE, ...) {

r <- dbSendQuery(conn, statement, ...)

on.exit(.jcall(r@stat, "V", "close"))
res <- dplyr::tbl_df(fetch(r, -1, block=1000))
if (type_convert) res <- readr::type_convert(res)

structure(
list(
cols = list(),
default = structure(list(), class = c("collector_guess", "collector"))
),
class = "col_spec"
) -> l

cols <- .jcall(r@md, "I", "getColumnCount")
nms <- c()

cts <- rep(0L, cols)
for (i in 1:cols) {
ct <- .jcall(r@md, "I", "getColumnType", i)
if (ct == .jt$CHAR) {
l$cols[[i]] <- col_character()
} else if (ct == .jt$NUMERIC) {
l$cols[[i]] <- col_double()
} else if (ct == .jt$DECIMAL) {
l$cols[[i]] <- col_double()
} else if (ct == .jt$INTEGER) {
l$cols[[i]] <- col_integer()
} else if (ct == .jt$SMALLINT) {
l$cols[[i]] <- col_integer()
} else if (ct == .jt$TINYINT) {
l$cols[[i]] <- col_integer()
} else if (ct == .jt$BIGINT) {
l$cols[[i]] <- col_double()
} else if (ct == .jt$FLOAT) {
l$cols[[i]] <- col_double()
} else if (ct == .jt$REAL) {
l$cols[[i]] <- col_double()
} else if (ct == .jt$DOUBLE) {
l$cols[[i]] <- col_double()
} else if (ct == .jt$VARCHAR) {
l$cols[[i]] <- col_character()
} else if (ct == .jt$DATE) {
l$cols[[i]] <- col_date()
} else if (ct == .jt$TIME) {
l$cols[[i]] <- col_time()
} else if (ct == .jt$TIMESTAMP) {
l$cols[[i]] <- col_datetime()
} else if (ct == .jt$BIT) {
l$cols[[i]] <- col_logical()
} else {
l$cols[[i]] <- col_character()
}
nms[i] <- .jcall(r@md, "S", "getColumnLabel", i)
}

l$cols <- stats::setNames(l$cols, nms)

res <- fetch(r, -1, block=1000)
res <- readr::type_convert(res, col_types = l)

class(res) <- c("tbl_df", "tbl", "data.frame")

res

}

)


R/metis-package.R → R/metis-lite-package.R View File

@@ -4,7 +4,7 @@
#' perform queries and retrieve query results. A lightweight 'RJDBC' implementation
#' is included along with an interface to the 'AWS' command-line utility.
#'
#' @name metis
#' @name metis.lite
#' @encoding UTF-8
#' @docType package
#' @author Bob Rudis (bob@@rud.is)

+ 23
- 0
R/sql_translate_env.R View File

@@ -1,6 +1,29 @@
#' @rdname Athena
#' @keywords internal
#' @export
db_data_type.AthenaConnection <- function(con, fields, ...) {
print("\n\n\ndb_data_type\n\n\n")
data_type <- function(x) {
switch(
class(x)[1],
integer64 = "BIGINT",
logical = "BOOLEAN",
integer = "INTEGER",
numeric = "DOUBLE",
factor = "CHARACTER",
character = "CHARACTER",
Date = "DATE",
POSIXct = "TIMESTAMP",
stop("Can't map type ", paste(class(x), collapse = "/"),
" to a supported database type.")
)
}
vapply(fields, data_type, character(1))
}

#' @rdname Athena
#' @keywords internal
#' @export
sql_translate_env.AthenaConnection <- function(con) {

x <- con


+ 2
- 2
R/zzz.R View File

@@ -1,4 +1,4 @@
.onLoad <- function(libname, pkgname) {
# rJava::.jpackage(pkgname, jars = "*", lib.loc = libname)
rJava::.jaddClassPath(dir(file.path(getwd(), "inst/java"), full.names = TRUE))
rJava::.jpackage(pkgname, jars = "*", lib.loc = libname)
# rJava::.jaddClassPath(dir(file.path(getwd(), "inst/java"), full.names = TRUE))
}

+ 3
- 0
man/Athena.Rd View File

@@ -2,11 +2,14 @@
% Please edit documentation in R/jdbc.r, R/sql_translate_env.R
\name{Athena}
\alias{Athena}
\alias{db_data_type.AthenaConnection}
\alias{sql_translate_env.AthenaConnection}
\title{AthenaJDBC}
\usage{
Athena(identifier.quote = "`")

\method{db_data_type}{AthenaConnection}(con, fields, ...)

\method{sql_translate_env}{AthenaConnection}(con)
}
\description{


man/metis.Rd → man/metis.lite.Rd View File

@@ -1,10 +1,10 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/metis-package.R
% Please edit documentation in R/metis-lite-package.R
\docType{package}
\encoding{UTF-8}
\name{metis}
\alias{metis}
\alias{metis-package}
\name{metis.lite}
\alias{metis.lite}
\alias{metis.lite-package}
\title{Helpers for Accessing and Querying Amazon Athena}
\description{
Methods are provides to connect to 'Amazon' 'Athena', lookup schemas/tables,

+ 1
- 1
man/use_credentials.Rd View File

@@ -1,5 +1,5 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/metis-package.R
% Please edit documentation in R/metis-lite-package.R
\name{use_credentials}
\alias{use_credentials}
\alias{read_credentials}


metis.Rproj → metis-lite.Rproj View File


+ 1
- 1
tests/test-all.R View File

@@ -1,2 +1,2 @@
library(testthat)
test_check("metis")
test_check("metis-lite")

tests/testthat/test-metis.R → tests/testthat/test-metis-lite.R View File


Loading…
Cancel
Save