Browse Source

another go at Travis

master
boB Rudis 4 years ago
parent
commit
c8450b643a
No known key found for this signature in database GPG Key ID: 1D7529BE14E2BBA9
  1. 9
      DESCRIPTION
  2. 1
      NAMESPACE
  3. 2
      R/metis-tidy-package.R
  4. 4
      R/sql_translate_env.R
  5. 5
      man/dbplyr-interface.Rd
  6. 14
      man/sql_translate_env.AthenaConnection.Rd
  7. 69
      tests/test-all.R
  8. 62
      tests/testthat/test-metis.tidy.R

9
DESCRIPTION

@ -14,13 +14,16 @@ Description: Methods are provided to use the 'metis' JDBC/DBI interface via
SystemRequirements: JDK 1.8+
License: MIT + file LICENSE
Suggests:
testthat,
covr
covr,
testthat
Depends:
R (>= 3.2.0),
metis
Imports:
DBI,
RJDBC,
rJava,
metis.jars,
metis,
dplyr,
dbplyr
RoxygenNote: 6.1.1

1
NAMESPACE

@ -5,6 +5,7 @@ S3method(sql_translate_env,AthenaConnection)
import(DBI)
import(dbplyr)
import(metis)
import(metis.jars)
importFrom(dplyr,db_data_type)
importFrom(dplyr,sql_translate_env)
importFrom(dplyr,tbl)

2
R/metis-tidy-package.R

@ -9,7 +9,7 @@
#' @keywords internal
#' @docType package
#' @author Bob Rudis (bob@@rud.is)
#' @import metis DBI dbplyr
#' @import metis.jars metis DBI dbplyr
#' @importFrom dplyr tbl db_data_type sql_translate_env
#' @references [Simba Athena JDBC Driver with SQL Connector Installation and Configuration Guide](https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC_2.0.6/docs/Simba+Athena+JDBC+Driver+Install+and+Configuration+Guide.pdf)
NULL

4
R/sql_translate_env.R

@ -27,7 +27,7 @@ db_data_type.AthenaConnection <- function(con, fields, ...) {
#' Translate R tridyverse functional idioms to Athena
#'
#' @rdname dbplyr-interface
#' @param con AthenaConnection
#' @export
sql_translate_env.AthenaConnection <- function(con) {
@ -107,4 +107,4 @@ sql_translate_env.AthenaConnection <- function(con) {
)
}
}

5
man/dbplyr-interface.Rd

@ -2,12 +2,9 @@
% Please edit documentation in R/sql_translate_env.R
\name{db_data_type.AthenaConnection}
\alias{db_data_type.AthenaConnection}
\alias{sql_translate_env.AthenaConnection}
\title{Convert R data type to Athena}
\usage{
\method{db_data_type}{AthenaConnection}(con, fields, ...)
\method{sql_translate_env}{AthenaConnection}(con)
}
\arguments{
\item{con}{Athena connection}
@ -18,6 +15,4 @@
}
\description{
Convert R data type to Athena
Translate R tridyverse functional idioms to Athena
}

14
man/sql_translate_env.AthenaConnection.Rd

@ -0,0 +1,14 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/sql_translate_env.R
\name{sql_translate_env.AthenaConnection}
\alias{sql_translate_env.AthenaConnection}
\title{Translate R tridyverse functional idioms to Athena}
\usage{
\method{sql_translate_env}{AthenaConnection}(con)
}
\arguments{
\item{con}{AthenaConnection}
}
\description{
Translate R tridyverse functional idioms to Athena
}

69
tests/test-all.R

@ -1,7 +1,64 @@
library(metis.jars)
library(metis)
library(dbplyr)
library(dplyr)
library(testthat)
options(tidyverse.quiet=TRUE)
test_check("metis.tidy")
library(dbplyr, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
library(dplyr, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
library(metis.jars, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
library(metis, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
library(metis.tidy, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE)
testthat::context("d[b]plyr ops work as expected")
Sys.setenv(
AWS_S3_STAGING_DIR = "s3://aws-athena-query-results-569593279821-us-east-1"
)
message("Making driver")
drv <- metis::Athena()
testthat::expect_is(drv, "AthenaDriver")
testthat::skip_on_cran()
message("Establishing connection")
if (identical(Sys.getenv("TRAVIS"), "true")) {
metis::dbConnect(
drv = drv,
Schema = "sampledb",
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1"
) -> con
} else {
metis::dbConnect(
drv = drv,
Schema = "sampledb",
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider",
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"),
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1",
) -> con
}
testthat::expect_is(con, "AthenaConnection")
message("Sourcing table")
elb_logs <- tbl(con, "elb_logs")
testthat::expect_is(elb_logs, "tbl_AthenaConnection")
message("Filtering and transforming")
filter(elb_logs, grepl("20", elbresponsecode)) %>%
mutate(
tsday = as.Date(substring(timestamp, 1L, 10L)),
host = url_extract_host(url),
proto_version = regexp_extract(protocol, "([[:digit:]\\.]+)"),
) %>%
select(tsday, host, receivedbytes, requestprocessingtime, proto_version) %>%
head(1) %>%
collect() -> out
testthat::expect_is(out$tsday, "Date")
testthat::expect_is(out$host, "character")
testthat::expect_is(out$receivedbytes, "integer64")
testthat::expect_is(out$requestprocessingtime, "numeric")
testthat::expect_is(out$proto_version, "character")

62
tests/testthat/test-metis.tidy.R

@ -1,62 +0,0 @@
context("d[b]plyr ops work as expected")
Sys.setenv(
AWS_S3_STAGING_DIR = "s3://aws-athena-query-results-569593279821-us-east-1"
)
library(metis)
library(dbplyr)
library(dplyr)
drv <- metis::Athena()
skip_on_cran()
if (identical(Sys.getenv("TRAVIS"), "true")) {
metis::dbConnect(
drv = drv,
Schema = "sampledb",
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1"
) -> con
} else {
metis::dbConnect(
drv = drv,
Schema = "sampledb",
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider",
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"),
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1",
) -> con
}
metis::dbConnect(
drv,
Schema = "sampledb",
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider",
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props")
) -> con
expect_is(con, "AthenaConnection")
elb_logs <- tbl(con, "elb_logs")
expect_is(elb_logs, "tbl_AthenaConnection")
filter(elb_logs, grepl("20", elbresponsecode)) %>%
mutate(
tsday = as.Date(substring(timestamp, 1L, 10L)),
host = url_extract_host(url),
proto_version = regexp_extract(protocol, "([[:digit:]\\.]+)"),
) %>%
select(tsday, host, receivedbytes, requestprocessingtime, proto_version) %>%
head(1) %>%
collect() -> out
expect_is(out$tsday, "Date")
expect_is(out$host, "character")
expect_is(out$receivedbytes, "integer64")
expect_is(out$requestprocessingtime, "numeric")
expect_is(out$proto_version, "character")
Loading…
Cancel
Save