From 2aa7c5daeb5bcd79864352799c556fc6fcf76197 Mon Sep 17 00:00:00 2001 From: hrbrmstr Date: Mon, 8 Jul 2019 14:25:16 -0400 Subject: [PATCH] initial commit --- .Rbuildignore | 1 + DESCRIPTION | 18 ++++-- LICENSE | 2 + LICENSE.md | 21 +++++++ NAMESPACE | 13 ++++- R/download-file.R | 143 +++++++++++++++++++++++++++++++++++++++++++++++ R/madhttr-package.R | 49 ++++++++++++++-- R/tidy-cert.R | 39 +++++++++++++ R/tidy-res.R | 56 +++++++++++++++++++ R/util.R | 1 + R/utils-pipe.R | 11 ++++ README.Rmd | 26 ++++++++- README.md | 88 +++++++++++++++++++++++++++++ man/GET.Rd | 13 +++++ man/POST.Rd | 11 ++++ man/download_file.Rd | 69 +++++++++++++++++++++++ man/download_ssl_cert.Rd | 11 ++++ man/madhttr.Rd | 10 ++-- man/nslookup.Rd | 11 ++++ man/pipe.Rd | 12 ++++ man/read_har.Rd | 16 ++++++ man/tidy_cert.Rd | 19 +++++++ man/tidy_response.Rd | 17 ++++++ 23 files changed, 639 insertions(+), 18 deletions(-) create mode 100644 LICENSE create mode 100644 LICENSE.md create mode 100644 R/download-file.R create mode 100644 R/tidy-cert.R create mode 100644 R/tidy-res.R create mode 100644 R/util.R create mode 100644 R/utils-pipe.R create mode 100644 man/GET.Rd create mode 100644 man/POST.Rd create mode 100644 man/download_file.Rd create mode 100644 man/download_ssl_cert.Rd create mode 100644 man/nslookup.Rd create mode 100644 man/pipe.Rd create mode 100644 man/read_har.Rd create mode 100644 man/tidy_cert.Rd create mode 100644 man/tidy_response.Rd diff --git a/.Rbuildignore b/.Rbuildignore index b28703c..da6311f 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -15,3 +15,4 @@ ^CONDUCT.*$ ^CODE.*$ ^\.gitlab-ci\.yml$ +^LICENSE\.md$ diff --git a/DESCRIPTION b/DESCRIPTION index 38bcb96..b9e9913 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,18 +1,26 @@ Package: madhttr Type: Package -Title: madhttr title goes here otherwise CRAN checks fail +Title: Tidy Helper Methods for Many Types of Unkempt Internet Metadata and Content Version: 0.1.0 Date: 2019-07-08 Authors@R: c( person("Bob", "Rudis", email = "bob@rud.is", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-5670-2640")) ) Maintainer: Bob Rudis -Description: A good description goes here otherwise CRAN checks fail. +Description: The 'httr', 'openssl', and 'HARtools' packages provide methods to retrieve + rich metadata and content from internet hosts but their return objects are quite + unkempt. Methods are provided to turn these objects into tidy data frames along with + other useful helper methods which augment functionality in these packages. URL: https://gitlab.com/hrbrmstr/madhttr BugReports: https://gitlab.com/hrbrmstr/madhttr/issues Encoding: UTF-8 -License: AGPL -Suggests: covr, tinytest +License: MIT + file LICENSE +Suggests: covr, tinytest, tibble Depends: R (>= 3.2.0) -Imports: httr, jsonlite +Imports: + httr, + openssl, + curl, + HARtools, + magrittr Roxygen: list(markdown = TRUE) RoxygenNote: 6.1.1 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b6a10f1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,2 @@ +YEAR: 2019 +COPYRIGHT HOLDER: Bob Rudis diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..c36552c --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# MIT License + +Copyright (c) 2019 Bob Rudis + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NAMESPACE b/NAMESPACE index 5b4b9ae..897ca50 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,4 +1,15 @@ # Generated by roxygen2: do not edit by hand +export("%>%") +export(GET) +export(POST) +export(download_file) +export(download_ssl_cert) +export(nslookup) +export(read_har) +export(tidy_cert) +export(tidy_response) import(httr) -importFrom(jsonlite,fromJSON) +importFrom(curl,nslookup) +importFrom(magrittr,"%>%") +importFrom(openssl,download_ssl_cert) diff --git a/R/download-file.R b/R/download-file.R new file mode 100644 index 0000000..9c4d302 --- /dev/null +++ b/R/download-file.R @@ -0,0 +1,143 @@ +possibly <- function(.f, otherwise, quiet = TRUE) { + force(otherwise) + function(...) { + tryCatch( + .f(...), + error = function(e) { + if (!quiet) + message("Error: ", e$message) + otherwise + }, + interrupt = function(e) { + stop("Terminated by user", call. = FALSE) + } + ) + } +} + +safe_GET <- possibly(GET, NULL, quiet = TRUE) + +#' Download file from the Internet (cache-aware) +#' +#' This is an alternative to [utils::download.file()] and a convenience wrapper for +#' [GET()] + [httr::write_disk()] to perform file downloads. +#' +#' Since this function uses [GET()], callers can pass in `httr` configuration +#' options to customize the behaviour of the download process (e.g. specify a `User-Agent` via +#' [user_agent()], set proxy config via [use_proxy()], etc.). +#' +#' The function is also "cache-aware" in the sense that you deliberately have to specify +#' `overwrite = TRUE` to force a re-download. This has the potential to save bandwidth +#' of both the caller and the site hosting files for download. +#' +#' @note While this function supports specifying multiple URLs and download paths it +#' does not perform concurrent downloads. +#' @param url the url(s) of the file to retrieve. If multiple URLs are provided then the same +#' number of `path`s must also be provided. +#' @param path Path(s) to save content to. If more than one `path` is specified then the same +#' number of `url`s must also be provided. THis parameter will be [path.expand()]ed. +#' @param overwrite Will only overwrite existing path if `TRUE`. +#' @param ... passed on to [GET()] +#' @return a data frame containing the `url`(s), `path`(s), cache status, and HTTP status code(s). +#' If there was an error downloading a file the path, status code, and HTTP status +#' columns will be `NA`. If the file was now re-downloaded the status code will be 399 +#' @seealso [GET()]; [write_disk()] +#' @export +#' @examples +#' tmp1 <- tempfile() +#' tmp2 <- tempfile() +#' tmp3 <- tempfile() +#' +#' download_file("https://google.com", tmp1) # downloads fine +#' download_file("https://google.com", tmp1) # doesn't re-download since it's cached +#' download_file("https://google.com", tmp1, overwrite = TRUE) # re-downloads (overwrites file) +#' download_file("https://google.com", tmp2) # re-downloads (new file) +#' download_file("badurl", tmp3) # handles major errors gracefully +#' +#' # multi-file example with no-caching +#' download_file( +#' c(rep("https://google.com", 2), "badurl"), +#' c(tmp1, tmp2, tmp3), +#' overwrite = TRUE +#' ) +#' +#' # multi-file example with caching +#' download_file( +#' c(rep("https://google.com", 2), "badurl"), +#' c(tmp1, tmp2, tmp3), +#' overwrite = FALSE +#' ) +download_file <- function(url, path, overwrite = FALSE, ...) { + + url <- as.character(url) + path <- as.character(path) + + if (length(url) != length(path)) { + stop("The lengths of the 'url' and 'path' parameters must be equal.", call.=FALSE) + } + + path <- path.expand(path) + + overwrite <- as.logical(overwrite) + stopifnot(length(overwrite) == 1) + + out <- vector("list", length = length(url)) + + for (idx in seq_along(url)) { + + u <- url[[idx]] + p <- path[[idx]] + + if (file.exists(p)) { + + if (overwrite) { # file exists but caller wants to re-download + res <- safe_GET(u, write_disk(p, overwrite = TRUE), ...) + if (is.null(res)) { + p <- NA_character_ + cache_used = FALSE + status <- NA_integer_ + } else { + cache_used <- FALSE + status <- status_code(res) + } + } else { # file exists but caller does not want to re-download + if (is.null(parse_url(u)[["hostname"]])) { # quick non-network test for invalid URL + p <- NA_character_ + cache_used = FALSE + status <- NA_integer_ + } else { + cache_used <- TRUE + status <- 399L + } + } + + } else { # file does not exist, so do the thing + + res <- safe_GET(u, write_disk(p, overwrite = overwrite), ...) + + if (is.null(res)) { + p <- NA_character_ + cache_used <- FALSE + status <- NA_integer_ + } else { + status <- status_code(res) + cache_used <- FALSE + } + + } + + out[[idx]] <- data.frame( + url = u, path = p, + status_code = status, + cache_used = cache_used, + stringsAsFactors = FALSE + ) + + } + + out <- do.call(rbind.data.frame, out) + class(out) <- c("tbl_df", "tbl", "data.frame") + + invisible(out) + +} diff --git a/R/madhttr-package.R b/R/madhttr-package.R index bde1471..b1fd75c 100644 --- a/R/madhttr-package.R +++ b/R/madhttr-package.R @@ -1,12 +1,49 @@ -#' ... -#' -#' - URL: -#' - BugReports: -#' +#' Tidy Helper Methods for Many Types of Unkempt Internet Metadata and Content +#' +#' The 'httr', 'openssl', and 'HARtools' packages provide methods to retrieve +#' rich metadata and content from internet hosts but their return objects are quite +#' unkempt. Methods are provided to turn these objects into tidy data frames along with +#' other useful helper methods which augment functionality in these packages. +#' #' @md #' @name madhttr #' @keywords internal #' @author Bob Rudis (bob@@rud.is) #' @import httr -#' @importFrom jsonlite fromJSON +#' @importFrom openssl download_ssl_cert +#' @importFrom curl nslookup "_PACKAGE" + +#' @name GET +#' @title GET a URL +#' @param url the URL of the page to retrieve +#' @param config,handle,... see [httr::GET()] +#' @export +NULL + +#' @name POST +#' @title POST file to a server +#' @param url,config,body,encode,handle,... see [httr::POST()] +#' @export +NULL + +#' Read HAR objects +#' +#' Snake-case re-exported alias for [HARtools::readHAR()]. +#' +#' @param har a string, list/URL or a file containing JSON HAR data +#' @param ... additional arguments +#' @export +read_har <- HARtools::readHAR + +#' @name nslookup +#' @title Lookup a hostname +#' @param host,ipv4_only,multiple,error see [curl::nslookup()] +#' @export +NULL + +#' @name download_ssl_cert +#' @title Doewnload X.509 certificates +#' @param host,port,ipv4_only see [openssl::download_ssl_cert()] +#' @export +NULL \ No newline at end of file diff --git a/R/tidy-cert.R b/R/tidy-cert.R new file mode 100644 index 0000000..851a79e --- /dev/null +++ b/R/tidy-cert.R @@ -0,0 +1,39 @@ +.tidy_one_cert <- function(.x) { + + data.frame( + subject = .x[["subject"]] %l0% NA_character_, + issuer = .x[["issuer"]] %l0% NA_character_, + algorithm = .x[["algorithm"]] %l0% NA_character_, + signature = I(list(.x[["signature"]])) %l0% I(list(raw())), + valid_start = .x[["validity"]][[1]] %l0% as.Date(NA), + valid_end = .x[["validity"]][[2]] %l0% as.Date(NA), + self_signed = .x[["self_signed"]] %l0% NA, + alt_names = I(list(.x[["alt_names"]] %l0% c())), + pub_key = I(list(.x[["pubkey"]]) %l0% list()), + stringsAsFactors = FALSE + ) + +} + +#' Turn an {openssl} downloaded SSL certificate into a tidy data frame +#' +#' @param .x a certificate list retrieved with [openssl::download_ssl_cert()] or +#' a single `cert` object. +#' @export +#' @examples +#' tidy_cert(openssl::download_ssl_cert("rud.is")) +#' tidy_cert(openssl::download_ssl_cert("r-project.org")[[2]]) +tidy_cert <- function(.x) { + + if (is.list(.x) && all(vapply(.x, inherits, FUN.VALUE = logical(1), what = "cert"))) { + out <- do.call(rbind.data.frame, lapply(.x, .tidy_one_cert)) + } else if (inherits(.x, "cert")) { + out <- .tidy_one_cert(.x) + } else { + stop("Input is not a list of certs or an individual cert.", call.=FALSE) + } + + class(out) <- c("tbl_df", "tbl", "data.frame") + out + +} diff --git a/R/tidy-res.R b/R/tidy-res.R new file mode 100644 index 0000000..1732dbe --- /dev/null +++ b/R/tidy-res.R @@ -0,0 +1,56 @@ +#' Turn an {httr} response object into a tidy data frame +#' +#' @param .x a `response` object retrieved with one of the [httr::VERB()] functions. +#' @export +#' @examples +#' tidy_response(httr::GET("https://rud.is")) +tidy_response <- function(.x) { + + `%l0%` <- function(a, b) if (length(a)) a else b + + if (length(.x[["times"]])) { + data.frame( + stage = names(.x[["times"]]), + value = as.numeric(.x[["times"]]), + stringsAsFactors = FALSE + ) -> times + } else { + times <- data.frame(stringsAsFactors = FALSE) + } + class(times) <- c("tbl_df", "tbl", "data.frame") + + if (length(.x[["all_headers"]])) { + + lapply(.x[["all_headers"]], function(.y) { + status <- .y[["status"]] %l0% NA_character_ + version <- .y[["version"]] %l0% NA_character_ + data.frame( + name = c(names(.y[["headers"]]), "status", "version"), + value = c(as.character(.y[["headers"]]), status, version), + stringsAsFactors = FALSE + ) + }) -> hdrs + + } else { + hdrs <- list() + } + + cookies <- .x[["cookies"]] %l0% data.frame(stringsAsFactors=FALSE) + class(cookies) <- c("tbl_df", "tbl", "data.frame") + + data.frame( + url = .x[["url"]] %l0% NA_character_, + status_code = .x[["status_code"]] %l0% NA_integer_, + date = .x[["date"]] %l0% as.POSIXct(NA), + headers = I(list(hdrs)), + cookies = I(list(cookies)), + content = I(list(.x[["content"]] %l0% c())), + times = I(list(times)), + stringsAsFactors = FALSE + ) -> out + + class(out) <- c("tbl_df", "tbl", "data.frame") + + out + +} \ No newline at end of file diff --git a/R/util.R b/R/util.R new file mode 100644 index 0000000..3327dd2 --- /dev/null +++ b/R/util.R @@ -0,0 +1 @@ +`%l0%` <- function(a, b) if (length(a)) a else b \ No newline at end of file diff --git a/R/utils-pipe.R b/R/utils-pipe.R new file mode 100644 index 0000000..fb8c818 --- /dev/null +++ b/R/utils-pipe.R @@ -0,0 +1,11 @@ +#' Pipe operator +#' +#' See \code{magrittr::\link[magrittr]{\%>\%}} for details. +#' +#' @name %>% +#' @rdname pipe +#' @keywords internal +#' @export +#' @importFrom magrittr %>% +#' @usage lhs \%>\% rhs +NULL diff --git a/README.Rmd b/README.Rmd index e259bc8..d69c71b 100644 --- a/README.Rmd +++ b/README.Rmd @@ -4,7 +4,7 @@ editor_options: chunk_output_type: inline --- ```{r pkg-knitr-opts, include=FALSE} -knitr:: sopts_chunk$set( +knitr::opts_chunk$set( collapse = TRUE, fig.retina = 2, message = FALSE, warning = FALSE ) options(width=120) @@ -16,12 +16,23 @@ options(width=120) # madhttr +Tidy Helper Methods for Many Types of Unkempt Internet Metadata and Content + ## Description +The 'httr', 'openssl', and 'HARtools' packages provide methods to retrieve +rich metadata and content from internet hosts but their return objects are quite +unkempt. Methods are provided to turn these objects into tidy data frames along with +other useful helper methods which augment functionality in these packages. + ## What's Inside The Tin The following functions are implemented: +```{r ingredients, results='asis', echo=FALSE} +hrbrpkghelpr::describe_ingredients() +``` + ## Installation ```{r install-ex, results='asis', echo = FALSE} @@ -32,12 +43,25 @@ hrbrpkghelpr::install_block() ```{r lib-ex} library(madhttr) +library(tibble) # for printing # current version packageVersion("madhttr") ``` +### Certifiable + +```{r certs} +tidy_cert(download_ssl_cert("r-project.org")) +``` + +### Responsive + +```{r resp} +tidy_response(GET("https://rud.is/b")) +``` + ## madhttr Metrics ```{r cloc, echo=FALSE} diff --git a/README.md b/README.md index 1bb7c6a..99c3fa8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,90 @@ + +[![Travis-CI Build +Status](https://travis-ci.org/hrbrmstr/madhttr.svg?branch=master)](https://travis-ci.org/hrbrmstr/madhttr) +[![Coverage +Status](https://codecov.io/gh/hrbrmstr/madhttr/branch/master/graph/badge.svg)](https://codecov.io/gh/hrbrmstr/madhttr) +[![CRAN\_Status\_Badge](https://www.r-pkg.org/badges/version/madhttr)](https://cran.r-project.org/package=madhttr) + # madhttr +Tidy Helper Methods for Many Types of Unkempt Internet Metadata and +Content + +## Description + +The ‘httr’, ‘openssl’, and ‘HARtools’ packages provide methods to +retrieve rich metadata and content from internet hosts but their return +objects are quite unkempt. Methods are provided to turn these objects +into tidy data frames along with other useful helper methods which +augment functionality in these packages. + +## What’s Inside The Tin + +The following functions are implemented: + + - `download_file`: Download file from the Internet (cache-aware) + - `download_ssl_cert`: Doewnload X.509 certificates + - `GET`: GET a URL + - `nslookup`: Lookup a hostname + - `POST`: POST file to a server + - `read_har`: Read HAR objects + - `tidy_cert`: Turn an openssl downloaded SSL certificate into a tidy + data frame + - `tidy_response`: Turn an httr response object into a tidy data frame + +## Installation + +``` r +devtools::install_git("https://git.sr.ht/~hrbrmstr/madhttr") +# or +devtools::install_gitlab("hrbrmstr/madhttr") +# or +devtools::install_bitbucket("hrbrmstr/madhttr") +``` + +## Usage + +``` r +library(madhttr) +library(tibble) # for printing + +# current version +packageVersion("madhttr") +## [1] '0.1.0' +``` + +### Certifiable + +``` r +tidy_cert(download_ssl_cert("r-project.org")) +## # A tibble: 4 x 9 +## subject issuer algorithm signature valid_start valid_end self_signed alt_names pub_key +## > > +## 1 CN=*.r-project.org,O… CN=COMODO RSA Domai… sha256With… < [256]> Aug 16 00:0… Aug 15 23… FALSE Feb 12 00:0… Feb 11 23… FALSE May 30 10:4… May 30 10… FALSE May 30 10:4… May 30 10… TRUE > > > > +## 1 https://rud.is/b/ 200 2019-07-08 18:10:38 < [60,585]> +``` + +## madhttr Metrics + +| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) | +| :--- | -------: | ---: | --: | ---: | ----------: | ---: | -------: | ---: | +| R | 7 | 0.88 | 149 | 0.93 | 43 | 0.66 | 112 | 0.74 | +| Rmd | 1 | 0.12 | 12 | 0.07 | 22 | 0.34 | 40 | 0.26 | + +## Code of Conduct + +Please note that this project is released with a [Contributor Code of +Conduct](CONDUCT.md). By participating in this project you agree to +abide by its terms. diff --git a/man/GET.Rd b/man/GET.Rd new file mode 100644 index 0000000..e974d91 --- /dev/null +++ b/man/GET.Rd @@ -0,0 +1,13 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/madhttr-package.R +\name{GET} +\alias{GET} +\title{GET a URL} +\arguments{ +\item{url}{the URL of the page to retrieve} + +\item{config, handle, ...}{see \code{\link[httr:GET]{httr::GET()}}} +} +\description{ +GET a URL +} diff --git a/man/POST.Rd b/man/POST.Rd new file mode 100644 index 0000000..4ad54fa --- /dev/null +++ b/man/POST.Rd @@ -0,0 +1,11 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/madhttr-package.R +\name{POST} +\alias{POST} +\title{POST file to a server} +\arguments{ +\item{url, config, body, encode, handle, ...}{see \code{\link[httr:POST]{httr::POST()}}} +} +\description{ +POST file to a server +} diff --git a/man/download_file.Rd b/man/download_file.Rd new file mode 100644 index 0000000..ec9c145 --- /dev/null +++ b/man/download_file.Rd @@ -0,0 +1,69 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/download-file.R +\name{download_file} +\alias{download_file} +\title{Download file from the Internet (cache-aware)} +\usage{ +download_file(url, path, overwrite = FALSE, ...) +} +\arguments{ +\item{url}{the url(s) of the file to retrieve. If multiple URLs are provided then the same +number of \code{path}s must also be provided.} + +\item{path}{Path(s) to save content to. If more than one \code{path} is specified then the same +number of \code{url}s must also be provided. THis parameter will be \code{\link[=path.expand]{path.expand()}}ed.} + +\item{overwrite}{Will only overwrite existing path if \code{TRUE}.} + +\item{...}{passed on to \code{\link[=GET]{GET()}}} +} +\value{ +a data frame containing the \code{url}(s), \code{path}(s), cache status, and HTTP status code(s). +If there was an error downloading a file the path, status code, and HTTP status +columns will be \code{NA}. If the file was now re-downloaded the status code will be 399 +} +\description{ +This is an alternative to \code{\link[utils:download.file]{utils::download.file()}} and a convenience wrapper for +\code{\link[=GET]{GET()}} + \code{\link[httr:write_disk]{httr::write_disk()}} to perform file downloads. +} +\details{ +Since this function uses \code{\link[=GET]{GET()}}, callers can pass in \code{httr} configuration +options to customize the behaviour of the download process (e.g. specify a \code{User-Agent} via +\code{\link[=user_agent]{user_agent()}}, set proxy config via \code{\link[=use_proxy]{use_proxy()}}, etc.). + +The function is also "cache-aware" in the sense that you deliberately have to specify +\code{overwrite = TRUE} to force a re-download. This has the potential to save bandwidth +of both the caller and the site hosting files for download. +} +\note{ +While this function supports specifying multiple URLs and download paths it +does not perform concurrent downloads. +} +\examples{ +tmp1 <- tempfile() +tmp2 <- tempfile() +tmp3 <- tempfile() + +download_file("https://google.com", tmp1) # downloads fine +download_file("https://google.com", tmp1) # doesn't re-download since it's cached +download_file("https://google.com", tmp1, overwrite = TRUE) # re-downloads (overwrites file) +download_file("https://google.com", tmp2) # re-downloads (new file) +download_file("badurl", tmp3) # handles major errors gracefully + +# multi-file example with no-caching +download_file( + c(rep("https://google.com", 2), "badurl"), + c(tmp1, tmp2, tmp3), + overwrite = TRUE +) + +# multi-file example with caching +download_file( + c(rep("https://google.com", 2), "badurl"), + c(tmp1, tmp2, tmp3), + overwrite = FALSE +) +} +\seealso{ +\code{\link[=GET]{GET()}}; \code{\link[=write_disk]{write_disk()}} +} diff --git a/man/download_ssl_cert.Rd b/man/download_ssl_cert.Rd new file mode 100644 index 0000000..6d36b12 --- /dev/null +++ b/man/download_ssl_cert.Rd @@ -0,0 +1,11 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/madhttr-package.R +\name{download_ssl_cert} +\alias{download_ssl_cert} +\title{Doewnload X.509 certificates} +\arguments{ +\item{host, port, ipv4_only}{see \code{\link[openssl:download_ssl_cert]{openssl::download_ssl_cert()}}} +} +\description{ +Doewnload X.509 certificates +} diff --git a/man/madhttr.Rd b/man/madhttr.Rd index d9ac890..3d852ab 100644 --- a/man/madhttr.Rd +++ b/man/madhttr.Rd @@ -4,12 +4,12 @@ \name{madhttr} \alias{madhttr} \alias{madhttr-package} -\title{...} +\title{Tidy Helper Methods for Many Types of Unkempt Internet Metadata and Content} \description{ -\itemize{ -\item URL: \url{https://gitlab.com/hrbrmstr/madhttr} -\item BugReports: \url{https://gitlab.com/hrbrmstr/madhttr/issues} -} +The 'httr', 'openssl', and 'HARtools' packages provide methods to retrieve +rich metadata and content from internet hosts but their return objects are quite +unkempt. Methods are provided to turn these objects into tidy data frames along with +other useful helper methods which augment functionality in these packages. } \seealso{ Useful links: diff --git a/man/nslookup.Rd b/man/nslookup.Rd new file mode 100644 index 0000000..3e7affa --- /dev/null +++ b/man/nslookup.Rd @@ -0,0 +1,11 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/madhttr-package.R +\name{nslookup} +\alias{nslookup} +\title{Lookup a hostname} +\arguments{ +\item{host, ipv4_only, multiple, error}{see \code{\link[curl:nslookup]{curl::nslookup()}}} +} +\description{ +Lookup a hostname +} diff --git a/man/pipe.Rd b/man/pipe.Rd new file mode 100644 index 0000000..b7daf6a --- /dev/null +++ b/man/pipe.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils-pipe.R +\name{\%>\%} +\alias{\%>\%} +\title{Pipe operator} +\usage{ +lhs \%>\% rhs +} +\description{ +See \code{magrittr::\link[magrittr]{\%>\%}} for details. +} +\keyword{internal} diff --git a/man/read_har.Rd b/man/read_har.Rd new file mode 100644 index 0000000..bc9dd61 --- /dev/null +++ b/man/read_har.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/madhttr-package.R +\name{read_har} +\alias{read_har} +\title{Read HAR objects} +\usage{ +read_har(har, ...) +} +\arguments{ +\item{har}{a string, list/URL or a file containing JSON HAR data} + +\item{...}{additional arguments} +} +\description{ +Snake-case re-exported alias for \code{\link[HARtools:readHAR]{HARtools::readHAR()}}. +} diff --git a/man/tidy_cert.Rd b/man/tidy_cert.Rd new file mode 100644 index 0000000..1f3116f --- /dev/null +++ b/man/tidy_cert.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/tidy-cert.R +\name{tidy_cert} +\alias{tidy_cert} +\title{Turn an {openssl} downloaded SSL certificate into a tidy data frame} +\usage{ +tidy_cert(.x) +} +\arguments{ +\item{.x}{a certificate list retrieved with \code{\link[openssl:download_ssl_cert]{openssl::download_ssl_cert()}} or +a single \code{cert} object.} +} +\description{ +Turn an {openssl} downloaded SSL certificate into a tidy data frame +} +\examples{ +tidy_cert(openssl::download_ssl_cert("rud.is")) +tidy_cert(openssl::download_ssl_cert("r-project.org")[[2]]) +} diff --git a/man/tidy_response.Rd b/man/tidy_response.Rd new file mode 100644 index 0000000..7bed4fa --- /dev/null +++ b/man/tidy_response.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/tidy-res.R +\name{tidy_response} +\alias{tidy_response} +\title{Turn an {httr} response object into a tidy data frame} +\usage{ +tidy_response(.x) +} +\arguments{ +\item{.x}{a \code{response} object retrieved with one of the \code{\link[httr:VERB]{httr::VERB()}} functions.} +} +\description{ +Turn an {httr} response object into a tidy data frame +} +\examples{ +tidy_response(httr::GET("https://rud.is")) +}