mirror of https://git.sr.ht/~hrbrmstr/metis-tidy
boB Rudis
5 years ago
8 changed files with 87 additions and 79 deletions
@ -0,0 +1,14 @@ |
|||
% Generated by roxygen2: do not edit by hand |
|||
% Please edit documentation in R/sql_translate_env.R |
|||
\name{sql_translate_env.AthenaConnection} |
|||
\alias{sql_translate_env.AthenaConnection} |
|||
\title{Translate R tridyverse functional idioms to Athena} |
|||
\usage{ |
|||
\method{sql_translate_env}{AthenaConnection}(con) |
|||
} |
|||
\arguments{ |
|||
\item{con}{AthenaConnection} |
|||
} |
|||
\description{ |
|||
Translate R tridyverse functional idioms to Athena |
|||
} |
@ -1,7 +1,64 @@ |
|||
library(metis.jars) |
|||
library(metis) |
|||
library(dbplyr) |
|||
library(dplyr) |
|||
library(testthat) |
|||
options(tidyverse.quiet=TRUE) |
|||
|
|||
test_check("metis.tidy") |
|||
library(dbplyr, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
|||
library(dplyr, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
|||
library(metis.jars, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
|||
library(metis, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
|||
library(metis.tidy, warn.conflicts = FALSE, quietly = TRUE, verbose = FALSE) |
|||
|
|||
testthat::context("d[b]plyr ops work as expected") |
|||
|
|||
Sys.setenv( |
|||
AWS_S3_STAGING_DIR = "s3://aws-athena-query-results-569593279821-us-east-1" |
|||
) |
|||
|
|||
message("Making driver") |
|||
drv <- metis::Athena() |
|||
testthat::expect_is(drv, "AthenaDriver") |
|||
|
|||
testthat::skip_on_cran() |
|||
|
|||
message("Establishing connection") |
|||
if (identical(Sys.getenv("TRAVIS"), "true")) { |
|||
|
|||
metis::dbConnect( |
|||
drv = drv, |
|||
Schema = "sampledb", |
|||
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1" |
|||
) -> con |
|||
|
|||
} else { |
|||
|
|||
metis::dbConnect( |
|||
drv = drv, |
|||
Schema = "sampledb", |
|||
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider", |
|||
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"), |
|||
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1", |
|||
) -> con |
|||
|
|||
} |
|||
|
|||
testthat::expect_is(con, "AthenaConnection") |
|||
|
|||
message("Sourcing table") |
|||
elb_logs <- tbl(con, "elb_logs") |
|||
|
|||
testthat::expect_is(elb_logs, "tbl_AthenaConnection") |
|||
|
|||
message("Filtering and transforming") |
|||
filter(elb_logs, grepl("20", elbresponsecode)) %>% |
|||
mutate( |
|||
tsday = as.Date(substring(timestamp, 1L, 10L)), |
|||
host = url_extract_host(url), |
|||
proto_version = regexp_extract(protocol, "([[:digit:]\\.]+)"), |
|||
) %>% |
|||
select(tsday, host, receivedbytes, requestprocessingtime, proto_version) %>% |
|||
head(1) %>% |
|||
collect() -> out |
|||
|
|||
testthat::expect_is(out$tsday, "Date") |
|||
testthat::expect_is(out$host, "character") |
|||
testthat::expect_is(out$receivedbytes, "integer64") |
|||
testthat::expect_is(out$requestprocessingtime, "numeric") |
|||
testthat::expect_is(out$proto_version, "character") |
|||
|
@ -1,62 +0,0 @@ |
|||
context("d[b]plyr ops work as expected") |
|||
|
|||
Sys.setenv( |
|||
AWS_S3_STAGING_DIR = "s3://aws-athena-query-results-569593279821-us-east-1" |
|||
) |
|||
|
|||
library(metis) |
|||
library(dbplyr) |
|||
library(dplyr) |
|||
|
|||
drv <- metis::Athena() |
|||
|
|||
skip_on_cran() |
|||
|
|||
if (identical(Sys.getenv("TRAVIS"), "true")) { |
|||
|
|||
metis::dbConnect( |
|||
drv = drv, |
|||
Schema = "sampledb", |
|||
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1" |
|||
) -> con |
|||
|
|||
} else { |
|||
|
|||
metis::dbConnect( |
|||
drv = drv, |
|||
Schema = "sampledb", |
|||
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider", |
|||
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props"), |
|||
S3OutputLocation = "s3://aws-athena-query-results-569593279821-us-east-1", |
|||
) -> con |
|||
|
|||
} |
|||
|
|||
metis::dbConnect( |
|||
drv, |
|||
Schema = "sampledb", |
|||
AwsCredentialsProviderClass = "com.simba.athena.amazonaws.auth.PropertiesFileCredentialsProvider", |
|||
AwsCredentialsProviderArguments = path.expand("~/.aws/athenaCredentials.props") |
|||
) -> con |
|||
|
|||
expect_is(con, "AthenaConnection") |
|||
|
|||
elb_logs <- tbl(con, "elb_logs") |
|||
|
|||
expect_is(elb_logs, "tbl_AthenaConnection") |
|||
|
|||
filter(elb_logs, grepl("20", elbresponsecode)) %>% |
|||
mutate( |
|||
tsday = as.Date(substring(timestamp, 1L, 10L)), |
|||
host = url_extract_host(url), |
|||
proto_version = regexp_extract(protocol, "([[:digit:]\\.]+)"), |
|||
) %>% |
|||
select(tsday, host, receivedbytes, requestprocessingtime, proto_version) %>% |
|||
head(1) %>% |
|||
collect() -> out |
|||
|
|||
expect_is(out$tsday, "Date") |
|||
expect_is(out$host, "character") |
|||
expect_is(out$receivedbytes, "integer64") |
|||
expect_is(out$requestprocessingtime, "numeric") |
|||
expect_is(out$proto_version, "character") |
Loading…
Reference in new issue