Ver código fonte

initial commit

master
boB Rudis 3 anos atrás
commit
a01ab3351f
Nenhuma chave conhecida encontrada para esta assinatura no banco de dados ID da chave GPG: 2A514A4997464560
25 arquivos alterados com 505 adições e 0 exclusões
  1. +10
    -0
      .Rbuildignore
  2. +1
    -0
      .codecov.yml
  3. +8
    -0
      .gitignore
  4. +31
    -0
      .travis.yml
  5. +22
    -0
      DESCRIPTION
  6. +14
    -0
      NAMESPACE
  7. +2
    -0
      NEWS.md
  8. +91
    -0
      R/jdbc.r
  9. +12
    -0
      R/metis-package.R
  10. +48
    -0
      R/metis.r
  11. +63
    -0
      README.Rmd
  12. +57
    -0
      README.md
  13. BIN
      inst/AthenaJDBC41-1.0.1.jar
  14. +11
    -0
      man/Athena.Rd
  15. +9
    -0
      man/AthenaConnection-class.Rd
  16. +9
    -0
      man/AthenaDriver-class.Rd
  17. +9
    -0
      man/AthenaResult-class.Rd
  18. +27
    -0
      man/athena_connect.Rd
  19. +15
    -0
      man/dbConnect-AthenaDriver-method.Rd
  20. +12
    -0
      man/dbGetQuery-AthenaConnection-character-method.Rd
  21. +12
    -0
      man/dbSendQuery-AthenaDriver-ANY-method.Rd
  22. +13
    -0
      man/metis.Rd
  23. +21
    -0
      metis.Rproj
  24. +2
    -0
      tests/test-all.R
  25. +6
    -0
      tests/testthat/test-metis.R

+ 10
- 0
.Rbuildignore Ver arquivo

@@ -0,0 +1,10 @@
^.*\.Rproj$
^\.Rproj\.user$
^\.travis\.yml$
^README\.*Rmd$
^README\.*html$
^NOTES\.*Rmd$
^NOTES\.*html$
^\.codecov\.yml$
^README_files$
^doc$

+ 1
- 0
.codecov.yml Ver arquivo

@@ -0,0 +1 @@
comment: false

+ 8
- 0
.gitignore Ver arquivo

@@ -0,0 +1,8 @@
.DS_Store
.Rproj.user
.Rhistory
.RData
.Rproj
src/*.o
src/*.so
src/*.dll

+ 31
- 0
.travis.yml Ver arquivo

@@ -0,0 +1,31 @@
language: r

warnings_are_errors: true

sudo: required

cache: packages

r:
- oldrel
- release
- devel

apt_packages:
- libv8-dev
- xclip

env:
global:
- CRAN: http://cran.rstudio.com

after_success:
- Rscript -e 'covr::codecov()'

notifications:
email:
- bob@rud.is
irc:
channels:
- "104.236.112.222#builds"
nick: travisci

+ 22
- 0
DESCRIPTION Ver arquivo

@@ -0,0 +1,22 @@
Package: metis
Type: Package
Title: Helpers for Accessing and Querying Amazon Athena
Version: 0.1.0
Date: 2017-05-16
Author: Bob Rudis (bob@rud.is)
Maintainer: Bob Rudis <bob@rud.is>
Description: Helpers for Accessing and Querying Amazon Athena. Including a lightweight RJDBC shim.
URL: https://github.com/hrbrmstr/metis
BugReports: https://github.com/hrbrmstr/metis/issues
License: AGPL
Suggests:
testthat,
covr
Depends:
R (>= 3.2.0),
RJDBC
Imports:
DBI,
dplyr,
ini
RoxygenNote: 6.0.1

+ 14
- 0
NAMESPACE Ver arquivo

@@ -0,0 +1,14 @@
# Generated by roxygen2: do not edit by hand

export(Athena)
export(athena_connect)
exportClasses(AthenaConnection)
exportClasses(AthenaDriver)
exportClasses(AthenaResult)
exportMethods(dbConnect)
exportMethods(dbGetQuery)
exportMethods(dbSendQuery)
import(DBI)
import(RJDBC)
import(dplyr)
import(ini)

+ 2
- 0
NEWS.md Ver arquivo

@@ -0,0 +1,2 @@
0.1.0
* Initial release

+ 91
- 0
R/jdbc.r Ver arquivo

@@ -0,0 +1,91 @@
#' AthenaJDBC
#'
#' @export
setClass("AthenaDriver", representation("JDBCDriver", identifier.quote="character", jdrv="jobjRef"))

#' AthenaJDBC
#'
#' @export
Athena <- function(identifier.quote='`') {
drv <- JDBC(driverClass="com.amazonaws.athena.jdbc.AthenaDriver",
system.file("AthenaJDBC41-1.0.1.jar", package="metis"),
identifier.quote="'")
return(as(drv, "AthenaDriver"))
}

#' AthenaJDBC
#'
#' @export
setMethod(

"dbConnect",
"AthenaDriver",

def = function(drv,
provider = "com.amazonaws.athena.jdbc.shaded.com.amazonaws.auth.EnvironmentVariableCredentialsProvider",
conn_string = 'jdbc:awsathena://athena.us-east-1.amazonaws.com:443/',
schema_name, ...) {

if (!is.null(provider)) {

jc <- callNextMethod(drv, conn_string,
s3_staging_dir=Sys.getenv("AWS_S3_STAGING_DIR"),
schema_name=schema_name,
aws_credentials_provider_class=provider, ...)

} else {

jc <- callNextMethod(drv,
'jdbc:awsathena://athena.us-east-1.amazonaws.com:443/',
s3_staging_dir=Sys.getenv("AWS_S3_STAGING_DIR"),
schema_name=schema_name,
user = Sys.getenv("AWS_ACCESS_KEY_ID"),
password = Sys.getenv("AWS_SECRET_ACCESS_KEY"))

}

return(as(jc, "AthenaConnection"))

}

)

#' AthenaJDBC
#'
#' @export
setClass("AthenaConnection", contains = "JDBCConnection")

#' AthenaJDBC
#'
#' @export
setClass("AthenaResult", contains = "JDBCResult")

#' AthenaJDBC
#'
#' @export
setMethod(

"dbSendQuery",
"AthenaDriver",

def = function(conn, statement, ...) {
return(as(callNextMethod(), "AthenaResult"))
}

)

#' AthenaJDBC
#'
#' @export
setMethod(

"dbGetQuery",
signature(conn="AthenaConnection", statement="character"),

def = function(conn, statement, ...) {
r <- dbSendQuery(conn, statement, ...)
on.exit(.jcall(r@stat, "V", "close"))
dplyr::tbl_df(fetch(r, -1, block=256))
}

)

+ 12
- 0
R/metis-package.R Ver arquivo

@@ -0,0 +1,12 @@
#' Helpers for Accessing and Querying Amazon Athena
#'
#' Including a lightweight RJDBC shim.
#'
#' @name metis
#' @docType package
#' @author Bob Rudis (bob@@rud.is)
#' @import RJDBC
#' @import DBI
#' @import dplyr
#' @import ini
NULL

+ 48
- 0
R/metis.r Ver arquivo

@@ -0,0 +1,48 @@
#' Make a JDBC connection to Athena
#'
#' Handles the up-front JDBC config
#'
#' For all connection types it is expected that you have the following environment variables
#' defined (a good place is `~/.Renviron`):
#'
#' - `AWS_S3_STAGING_DIR`: the name of the S3 bucket where Athena can write stuff
#' - `AWS_PROFILE`: the AWS profile ID in `~/.aws/credentials` (defaults to `default` if not present)
#'
#' For `simple` == `FALSE` the expectation is that you're working with a managed
#' `~/.aws/credentials` file.
#'
#' @md
#' @param default_schema def sch
#' @param simple simple
#' @export
athena_connect <- function(default_schema, simple=FALSE) {

athena_jdbc <- Athena()

aws_config <- ini::read.ini(path.expand("~/.aws/credentials"))
aws_profile <- aws_config[Sys.getenv("AWS_PROFILE", "default")][[1]]

Sys.unsetenv("AWS_ACCESS_KEY_ID")
Sys.unsetenv("AWS_SECRET_ACCESS_KEY")

Sys.setenv(AWS_ACCESS_KEY_ID = aws_profile$aws_access_key_id)
Sys.setenv(AWS_SECRET_ACCESS_KEY = aws_profile$aws_secret_access_key)

con <- NULL

if (!simple) {

Sys.unsetenv("AWS_SESSION_TOKEN")
Sys.setenv(AWS_SESSION_TOKEN = aws_profile$aws_session_token)

con <- dbConnect(athena_jdbc, schema_name = default_schema)

} else {

con <- dbConnect(athena_jdbc, provider = NULL, schema_name = default_schema)

}

con

}

+ 63
- 0
README.Rmd Ver arquivo

@@ -0,0 +1,63 @@
---
output: rmarkdown::github_document
---

[`metis`](https://en.wikipedia.org/wiki/Metis_(mythology)) : Helpers for Accessing and Querying Amazon Athena

Including a lightweight RJDBC shim.

![](https://upload.wikimedia.org/wikipedia/commons/thumb/5/53/Winged_goddess_Louvre_F32.jpg/300px-Winged_goddess_Louvre_F32.jpg)

THIS IS SUPER ALPHA QUALITY. NOTHING TO SEE HERE. MOVE ALONG.

The goal will be to get around enough of the "gotchas" that are preventing raw RJDBC Athena
connecitons from "just working" with `dplyr` v0.6.0+ and also get around the [`fetchSize` problem](https://www.reddit.com/r/aws/comments/6aq22b/fetchsize_limit/) without having to not use `dbGetQuery()`.

It will also support more than the vanilla id/secret auth mechism (it currently support the default basic auth and temp token auth, the latter via environment variables).

See the **Usage** section for an example.

The following functions are implemented:

- `athena_connect`: Make a JDBC connection to Athena (this returns an `AthenaConnection` object which is a super-class of it's RJDBC vanilla counterpart)
- `Athena`: AthenaJDBC`
- `AthenaConnection-class`: AthenaJDBC
- `AthenaDriver-class`: AthenaJDBC
- `AthenaResult-class`: AthenaJDBC
- `dbConnect-method`: AthenaJDBC
- `dbGetQuery-method`: AthenaJDBC
- `dbSendQuery-method`: AthenaJDBC

### Installation

```{r eval=FALSE}
devtools::install_github("hrbrmstr/metis")
```

```{r message=FALSE, warning=FALSE, error=FALSE, include=FALSE}
options(width=120)
```

### Usage

```{r message=FALSE, warning=FALSE, error=FALSE}
library(metis)
library(dplyr)

# current verison
packageVersion("metis")
```

```{r message=FALSE, warning=FALSE, error=FALSE, eval=FALSE}
ath <- athena_connect("your_schema_name")

res <- dbGetQuery(ath, "
SELECT format_datetime(timestamp, 'yyyy-MM-dd HH:00:00') timestamp,
port as field, count(port) cnt_field FROM your_schema_name.your_table_name
WHERE CONTAINS(ARRAY['201705'], date)
AND port IN (445, 139, 3389)
AND timestamp > date '2017-05-01'
AND timestamp <= date '2017-05-22'
GROUP BY format_datetime(timestamp, 'yyyy-MM-dd HH:00:00'), port LIMIT 1000000
")
```

+ 57
- 0
README.md Ver arquivo

@@ -0,0 +1,57 @@

[`metis`](https://en.wikipedia.org/wiki/Metis_(mythology)) : Helpers for Accessing and Querying Amazon Athena

Including a lightweight RJDBC shim.

![](https://upload.wikimedia.org/wikipedia/commons/thumb/5/53/Winged_goddess_Louvre_F32.jpg/300px-Winged_goddess_Louvre_F32.jpg)

THIS IS SUPER ALPHA QUALITY. NOTHING TO SEE HERE. MOVE ALONG.

The goal will be to get around enough of the "gotchas" that are preventing raw RJDBC Athena connecitons from "just working" with `dplyr` v0.6.0+ and also get around the [`fetchSize` problem](https://www.reddit.com/r/aws/comments/6aq22b/fetchsize_limit/) without having to not use `dbGetQuery()`.

It will also support more than the vanilla id/secret auth mechism (it currently support the default basic auth and temp token auth, the latter via environment variables).

See the **Usage** section for an example.

The following functions are implemented:

- `athena_connect`: Make a JDBC connection to Athena (this returns an `AthenaConnection` object which is a super-class of it's RJDBC vanilla counterpart)
- `Athena`: AthenaJDBC\`
- `AthenaConnection-class`: AthenaJDBC
- `AthenaDriver-class`: AthenaJDBC
- `AthenaResult-class`: AthenaJDBC
- `dbConnect-method`: AthenaJDBC
- `dbGetQuery-method`: AthenaJDBC
- `dbSendQuery-method`: AthenaJDBC

### Installation

``` r
devtools::install_github("hrbrmstr/metis")
```

### Usage

``` r
library(metis)
library(dplyr)

# current verison
packageVersion("metis")
```

## [1] '0.1.0'

``` r
ath <- athena_connect("your_schema_name")

res <- dbGetQuery(ath, "
SELECT format_datetime(timestamp, 'yyyy-MM-dd HH:00:00') timestamp,
port as field, count(port) cnt_field FROM your_schema_name.your_table_name
WHERE CONTAINS(ARRAY['201705'], date)
AND port IN (445, 139, 3389)
AND timestamp > date '2017-05-01'
AND timestamp <= date '2017-05-22'
GROUP BY format_datetime(timestamp, 'yyyy-MM-dd HH:00:00'), port LIMIT 1000000
")
```

BIN
inst/AthenaJDBC41-1.0.1.jar Ver arquivo


+ 11
- 0
man/Athena.Rd Ver arquivo

@@ -0,0 +1,11 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/jdbc.r
\name{Athena}
\alias{Athena}
\title{AthenaJDBC}
\usage{
Athena(identifier.quote = "`")
}
\description{
AthenaJDBC
}

+ 9
- 0
man/AthenaConnection-class.Rd Ver arquivo

@@ -0,0 +1,9 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/jdbc.r
\docType{class}
\name{AthenaConnection-class}
\alias{AthenaConnection-class}
\title{AthenaJDBC}
\description{
AthenaJDBC
}

+ 9
- 0
man/AthenaDriver-class.Rd Ver arquivo

@@ -0,0 +1,9 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/jdbc.r
\docType{class}
\name{AthenaDriver-class}
\alias{AthenaDriver-class}
\title{AthenaJDBC}
\description{
AthenaJDBC
}

+ 9
- 0
man/AthenaResult-class.Rd Ver arquivo

@@ -0,0 +1,9 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/jdbc.r
\docType{class}
\name{AthenaResult-class}
\alias{AthenaResult-class}
\title{AthenaJDBC}
\description{
AthenaJDBC
}

+ 27
- 0
man/athena_connect.Rd Ver arquivo

@@ -0,0 +1,27 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/metis.r
\name{athena_connect}
\alias{athena_connect}
\title{Make a JDBC connection to Athena}
\usage{
athena_connect(default_schema, simple = FALSE)
}
\arguments{
\item{default_schema}{def sch}

\item{simple}{simple}
}
\description{
Handles the up-front JDBC config
}
\details{
For all connection types it is expected that you have the following environment variables
defined (a good place is \code{~/.Renviron}):
\itemize{
\item \code{AWS_S3_STAGING_DIR}: the name of the S3 bucket where Athena can write stuff
\item \code{AWS_PROFILE}: the AWS profile ID in \code{~/.aws/credentials} (defaults to \code{default} if not present)
}

For \code{simple} == \code{FALSE} the expectation is that you're working with a managed
\code{~/.aws/credentials} file.
}

+ 15
- 0
man/dbConnect-AthenaDriver-method.Rd Ver arquivo

@@ -0,0 +1,15 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/jdbc.r
\docType{methods}
\name{dbConnect,AthenaDriver-method}
\alias{dbConnect,AthenaDriver-method}
\title{AthenaJDBC}
\usage{
\S4method{dbConnect}{AthenaDriver}(drv,
provider = "com.amazonaws.athena.jdbc.shaded.com.amazonaws.auth.EnvironmentVariableCredentialsProvider",
conn_string = "jdbc:awsathena://athena.us-east-1.amazonaws.com:443/",
schema_name, ...)
}
\description{
AthenaJDBC
}

+ 12
- 0
man/dbGetQuery-AthenaConnection-character-method.Rd Ver arquivo

@@ -0,0 +1,12 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/jdbc.r
\docType{methods}
\name{dbGetQuery,AthenaConnection,character-method}
\alias{dbGetQuery,AthenaConnection,character-method}
\title{AthenaJDBC}
\usage{
\S4method{dbGetQuery}{AthenaConnection,character}(conn, statement, ...)
}
\description{
AthenaJDBC
}

+ 12
- 0
man/dbSendQuery-AthenaDriver-ANY-method.Rd Ver arquivo

@@ -0,0 +1,12 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/jdbc.r
\docType{methods}
\name{dbSendQuery,AthenaDriver,ANY-method}
\alias{dbSendQuery,AthenaDriver,ANY-method}
\title{AthenaJDBC}
\usage{
\S4method{dbSendQuery}{AthenaDriver,ANY}(conn, statement, ...)
}
\description{
AthenaJDBC
}

+ 13
- 0
man/metis.Rd Ver arquivo

@@ -0,0 +1,13 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/metis-package.R
\docType{package}
\name{metis}
\alias{metis}
\alias{metis-package}
\title{Helpers for Accessing and Querying Amazon Athena}
\description{
Including a lightweight RJDBC shim.
}
\author{
Bob Rudis (bob@rud.is)
}

+ 21
- 0
metis.Rproj Ver arquivo

@@ -0,0 +1,21 @@
Version: 1.0

RestoreWorkspace: Default
SaveWorkspace: Default
AlwaysSaveHistory: Default

EnableCodeIndexing: Yes
UseSpacesForTab: Yes
NumSpacesForTab: 2
Encoding: UTF-8

RnwWeave: Sweave
LaTeX: pdfLaTeX

StripTrailingWhitespace: Yes

BuildType: Package
PackageUseDevtools: Yes
PackageInstallArgs: --no-multiarch --with-keep.source
PackageBuildArgs: --resave-data
PackageRoxygenize: rd,collate,namespace

+ 2
- 0
tests/test-all.R Ver arquivo

@@ -0,0 +1,2 @@
library(testthat)
test_check("metis")

+ 6
- 0
tests/testthat/test-metis.R Ver arquivo

@@ -0,0 +1,6 @@
context("basic functionality")
test_that("we can do something", {

#expect_that(some_function(), is_a("data.frame"))

})

Carregando…
Cancelar
Salvar