mirror of https://git.sr.ht/~hrbrmstr/metis-tidy
boB Rudis
5 years ago
8 changed files with 87 additions and 79 deletions
@ -0,0 +1,14 @@ |
|||||
|
% Generated by roxygen2: do not edit by hand |
||||
|
% Please edit documentation in R/sql_translate_env.R |
||||
|
\name{sql_translate_env.AthenaConnection} |
||||
|
\alias{sql_translate_env.AthenaConnection} |
||||
|
\title{Translate R tridyverse functional idioms to Athena} |
||||
|
\usage{ |
||||
|
\method{sql_translate_env}{AthenaConnection}(con) |
||||
|
} |
||||
|
\arguments{ |
||||
|
\item{con}{AthenaConnection} |
||||
|
} |
||||
|
\description{ |
||||
|
Translate R tridyverse functional idioms to Athena |
||||
|
} |
@ -1,7 +1,64 @@ |
|||||
library(metis.jars) |
options(tidyverse.quiet=TRUE) |
||||
library(metis) |
|
||||
library(dbplyr) |
|
||||
library(dplyr) |
|
||||
library(testthat) |
|
||||
|
|
||||
test_check("metis.tidy") |
library(dbplyr, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
||||
|
library(dplyr, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
||||
|
library(metis.jars, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
||||
|
library(metis, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
||||
|
library(metis.tidy, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
||||
|
|
||||
|
testthat::context("d[b]plyr ops work as expected") |
||||
|
|
||||
|
Sys.setenv( |
||||
|
AWS_S3_STAGING_DIR = "s3://aws-athena-query-results-569593279821-us-east-1" |
||||
|
) |
||||
|
|
||||
|
message("Making driver") |
||||
|
drv <- metis::Athena() |
||||
|
testthat::expect_is(drv, "AthenaDriver") |
||||
|
|
||||
|
testthat::skip_on_cran() |
||||
|
|
||||
|
message("Establishing connection") |
||||
|
if (identical(Sys.getenv("TRAVIS"), "true")) { |
||||
|
|
||||
|
metis::dbConnect( |
||||
|
drv = drv, |
||||
|
Schema = "sampledb", |
||||
|
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1" |
||||
|
) -> con |
||||
|
|
||||
|
} else { |
||||
|
|
||||
|
metis::dbConnect( |
||||
|
drv = drv, |
||||
|
Schema = "sampledb", |
||||
|
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider", |
||||
|
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"), |
||||
|
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1", |
||||
|
) -> con |
||||
|
|
||||
|
} |
||||
|
|
||||
|
testthat::expect_is(con, "AthenaConnection") |
||||
|
|
||||
|
message("Sourcing table") |
||||
|
elb_logs <- tbl(con, "elb_logs") |
||||
|
|
||||
|
testthat::expect_is(elb_logs, "tbl_AthenaConnection") |
||||
|
|
||||
|
message("Filtering and transforming") |
||||
|
filter(elb_logs, grepl("20", elbresponsecode)) %>% |
||||
|
mutate( |
||||
|
tsday = as.Date(substring(timestamp, 1L, 10L)), |
||||
|
host = url_extract_host(url), |
||||
|
proto_version = regexp_extract(protocol, "([[:digit:]\\.]+)"), |
||||
|
) %>% |
||||
|
select(tsday, host, receivedbytes, requestprocessingtime, proto_version) %>% |
||||
|
head(1) %>% |
||||
|
collect() -> out |
||||
|
|
||||
|
testthat::expect_is(out$tsday, "Date") |
||||
|
testthat::expect_is(out$host, "character") |
||||
|
testthat::expect_is(out$receivedbytes, "integer64") |
||||
|
testthat::expect_is(out$requestprocessingtime, "numeric") |
||||
|
testthat::expect_is(out$proto_version, "character") |
||||
|
@ -1,62 +0,0 @@ |
|||||
context("d[b]plyr ops work as expected") |
|
||||
|
|
||||
Sys.setenv( |
|
||||
AWS_S3_STAGING_DIR = "s3://aws-athena-query-results-569593279821-us-east-1" |
|
||||
) |
|
||||
|
|
||||
library(metis) |
|
||||
library(dbplyr) |
|
||||
library(dplyr) |
|
||||
|
|
||||
drv <- metis::Athena() |
|
||||
|
|
||||
skip_on_cran() |
|
||||
|
|
||||
if (identical(Sys.getenv("TRAVIS"), "true")) { |
|
||||
|
|
||||
metis::dbConnect( |
|
||||
drv = drv, |
|
||||
Schema = "sampledb", |
|
||||
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1" |
|
||||
) -> con |
|
||||
|
|
||||
} else { |
|
||||
|
|
||||
metis::dbConnect( |
|
||||
drv = drv, |
|
||||
Schema = "sampledb", |
|
||||
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider", |
|
||||
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"), |
|
||||
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1", |
|
||||
) -> con |
|
||||
|
|
||||
} |
|
||||
|
|
||||
metis::dbConnect( |
|
||||
drv, |
|
||||
Schema = "sampledb", |
|
||||
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider", |
|
||||
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props") |
|
||||
) -> con |
|
||||
|
|
||||
expect_is(con, "AthenaConnection") |
|
||||
|
|
||||
elb_logs <- tbl(con, "elb_logs") |
|
||||
|
|
||||
expect_is(elb_logs, "tbl_AthenaConnection") |
|
||||
|
|
||||
filter(elb_logs, grepl("20", elbresponsecode)) %>% |
|
||||
mutate( |
|
||||
tsday = as.Date(substring(timestamp, 1L, 10L)), |
|
||||
host = url_extract_host(url), |
|
||||
proto_version = regexp_extract(protocol, "([[:digit:]\\.]+)"), |
|
||||
) %>% |
|
||||
select(tsday, host, receivedbytes, requestprocessingtime, proto_version) %>% |
|
||||
head(1) %>% |
|
||||
collect() -> out |
|
||||
|
|
||||
expect_is(out$tsday, "Date") |
|
||||
expect_is(out$host, "character") |
|
||||
expect_is(out$receivedbytes, "integer64") |
|
||||
expect_is(out$requestprocessingtime, "numeric") |
|
||||
expect_is(out$proto_version, "character") |
|
Loading…
Reference in new issue