diff --git a/DESCRIPTION b/DESCRIPTION index 74ea5cc..07c97fe 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: httr2curl Type: Package -Title: httr2curl Title Goes Here Otherwise CRAN Checks Fail +Title: Turn 'httr' Web Requests Into 'curl' Command Line Calls Version: 0.1.0 Date: 2021-01-27 Authors@R: c( @@ -8,17 +8,21 @@ Authors@R: c( comment = c(ORCID = "0000-0001-5670-2640")) ) Maintainer: Bob Rudis -Description: A good description goes here otherwise CRAN checks fail. +Description: When provided with an 'httr' "VERB" call, the request will be made + and 'libcurl' headers will be captured and post-processed to provide a working + 'curl' command line call. URL: https://git.rud.is/hrbrmstr/httr2curl BugReports: https://git.rud.is/hrbrmstr/httr2curl/issues Encoding: UTF-8 -License: AGPL +License: MIT + file LICENSE +SystemRequirements: perl Suggests: covr, tinytest Depends: R (>= 3.6.0) Imports: httr, - jsonlite + processx, + magrittr Roxygen: list(markdown = TRUE) RoxygenNote: 7.1.1 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f51d1b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,2 @@ +YEAR: 2021 +COPYRIGHT HOLDER: Bob Rudis diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..c2304a0 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# MIT License + +Copyright (c) 2021 Bob Rudis + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NAMESPACE b/NAMESPACE index 5b4b9ae..9e3cef1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,4 +1,7 @@ # Generated by roxygen2: do not edit by hand +export("%>%") +export(h2c) import(httr) -importFrom(jsonlite,fromJSON) +import(processx) +importFrom(magrittr,"%>%") diff --git a/R/h2c.R b/R/h2c.R new file mode 100644 index 0000000..da832b3 --- /dev/null +++ b/R/h2c.R @@ -0,0 +1,48 @@ +#' Convert an 'httr' call to 'curl' command line +#' +#' @param complete_httr_verb_call wrap an `httr` `VERB` call with this function +#' and it will return the text of a working `curl` command line +#' @export +#' @examples \dontrun{ +#' h2c( +#' httr::GET( +#' url = "https://rud.is/", +#' httr::user_agent(splashr::ua_apple_tv), +#' query = list( +#' a = "b", +#' c = 1 +#' ) +#' ) +#' ) +#' } +h2c <- function(complete_httr_verb_call) { + + ƒ_call <- substitute(complete_httr_verb_call) + + perl <- find_perl() + + args <- system.file("bin", "h2c.pl", package = "httr2curl") + + capture.output( + capture.output( + httr::with_verbose( + eval(ƒ_call) + ), type = "message") -> res + ) -> junk + + out <- tempfile() + + res[grepl("^->", res)] %>% + sub("^-> ", "", .) %>% + paste0(collapse = "\n") %>% + writeLines(out) + + processx::run( + command = perl, + args = args, + stdin = out + ) -> res + + res$stdout + +} \ No newline at end of file diff --git a/R/httr2curl-package.R b/R/httr2curl-package.R index 8ac42e0..ab0d5a2 100644 --- a/R/httr2curl-package.R +++ b/R/httr2curl-package.R @@ -1,9 +1,12 @@ -#' ... -#' +#' Turn 'httr' Web Requests Into 'curl' Command Line Calls +#' +#' When provided with an 'httr' "VERB" call, the request will be made +#' and 'libcurl' headers will be captured and post-processed to provide a working +#' 'curl' command line call. +#' #' @md #' @name httr2curl #' @keywords internal #' @author Bob Rudis (bob@@rud.is) -#' @import httr -#' @importFrom jsonlite fromJSON +#' @import httr processx "_PACKAGE" diff --git a/R/utils-pipe.R b/R/utils-pipe.R new file mode 100644 index 0000000..e79f3d8 --- /dev/null +++ b/R/utils-pipe.R @@ -0,0 +1,11 @@ +#' Pipe operator +#' +#' See \code{magrittr::\link[magrittr:pipe]{\%>\%}} for details. +#' +#' @name %>% +#' @rdname pipe +#' @keywords internal +#' @export +#' @importFrom magrittr %>% +#' @usage lhs \%>\% rhs +NULL diff --git a/R/utils.R b/R/utils.R new file mode 100644 index 0000000..c747aa2 --- /dev/null +++ b/R/utils.R @@ -0,0 +1,14 @@ +find_perl <- function() { + + perl <- Sys.which("perl") + + if (perl == "") { + stop( + "Cannot find 'perl'. httr2curl requires perl to be installed and on the PATH.", + call. = FALSE + ) + } + + return(perl) + +} \ No newline at end of file diff --git a/README.Rmd b/README.Rmd index 30917b2..8ddf128 100644 --- a/README.Rmd +++ b/README.Rmd @@ -3,6 +3,7 @@ output: rmarkdown::github_document editor_options: chunk_output_type: console --- + ```{r pkg-knitr-opts, include=FALSE} hrbrpkghelpr::global_opts() ``` @@ -36,7 +37,19 @@ library(httr2curl) # current version packageVersion("httr2curl") +``` +```{r ex01} +h2c( + httr::GET( + url = "https://rud.is/", + httr::user_agent(splashr::ua_apple_tv), + query = list( + a = "b", + c = 1 + ) + ) +) ``` ## httr2curl Metrics @@ -47,5 +60,6 @@ cloc::cloc_pkg_md() ## Code of Conduct -Please note that this project is released with a Contributor Code of Conduct. -By participating in this project you agree to abide by its terms. +Please note that this project is released with a Contributor Code of +Conduct. By participating in this project you agree to abide by its +terms. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1abe2da --- /dev/null +++ b/README.md @@ -0,0 +1,80 @@ + +[![Project Status: Active – The project has reached a stable, usable +state and is being actively +developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) +[![Signed +by](https://img.shields.io/badge/Keybase-Verified-brightgreen.svg)](https://keybase.io/hrbrmstr) +![Signed commit +%](https://img.shields.io/badge/Signed_Commits-100%25-lightgrey.svg) +[![R-CMD-check](https://github.com/hrbrmstr/httr2curl/workflows/R-CMD-check/badge.svg)](https://github.com/hrbrmstr/httr2curl/actions?query=workflow%3AR-CMD-check) +[![Linux build +Status](https://travis-ci.org/hrbrmstr/httr2curl.svg?branch=master)](https://travis-ci.org/hrbrmstr/httr2curl) +![Minimal R +Version](https://img.shields.io/badge/R%3E%3D-3.6.0-blue.svg) +![License](https://img.shields.io/badge/License-MIT-blue.svg) + +# httr2curl + +Turn ‘httr’ Web Requests Into ‘curl’ Command Line Calls + +## Description + +When provided with an ‘httr’ “VERB” call, the request will be made and +‘libcurl’ headers will be captured and post-processed to provide a +working ‘curl’ command line call. + +## What’s Inside The Tin + +The following functions are implemented: + +- `h2c`: Convert an ‘httr’ call to ‘curl’ command line + +## Installation + +``` r +remotes::install_git("https://git.rud.is/hrbrmstr/httr2curl.git") +``` + +NOTE: To use the ‘remotes’ install options you will need to have the +[{remotes} package](https://github.com/r-lib/remotes) installed. + +## Usage + +``` r +library(httr2curl) + +# current version +packageVersion("httr2curl") +## [1] '0.1.0' +``` + +``` r +h2c( + httr::GET( + url = "https://rud.is/", + httr::user_agent(splashr::ua_apple_tv), + query = list( + a = "b", + c = 1 + ) + ) +) +## [1] "curl --http2 --header \"Accept: application/json, text/xml, application/xml, */*\" --compressed --user-agent \"AppleTV6,2/11.1\" \"https://rud.is/?a=b&c=1\"\n" +``` + +## httr2curl Metrics + +| Lang | \# Files | (%) | LoC | (%) | Blank lines | (%) | \# Lines | (%) | +|:-----|---------:|-----:|----:|-----:|------------:|-----:|---------:|-----:| +| R | 5 | 0.36 | 37 | 0.24 | 15 | 0.23 | 38 | 0.27 | +| YAML | 1 | 0.07 | 22 | 0.14 | 2 | 0.03 | 2 | 0.01 | +| Rmd | 1 | 0.07 | 18 | 0.12 | 16 | 0.24 | 31 | 0.22 | +| SUM | 7 | 0.50 | 77 | 0.50 | 33 | 0.50 | 71 | 0.50 | + +clock Package Metrics for httr2curl + +## Code of Conduct + +Please note that this project is released with a Contributor Code of +Conduct. By participating in this project you agree to abide by its +terms. diff --git a/inst/bin/h2c.pl b/inst/bin/h2c.pl new file mode 100755 index 0000000..5230cfa --- /dev/null +++ b/inst/bin/h2c.pl @@ -0,0 +1,426 @@ +#!/usr/bin/perl + +use MIME::Base64; + +sub usage { + print "h2c.pl [options] < file \n", + " -a Allow curl's default headers\n", + " -d Output man page HTML links after command line\n", + " -h Show short help\n", + " -H Output HTTP generated URLs instead\n", + " -i Ignore HTTP version\n", + " --libcurl Output libcurl code instead\n", + " -n Output notes after command line\n", + " -s Use short command line options\n", + " -v Add a verbose option to the command line\n"; + exit; +} + +sub manpage { + my ($p, $n, $desc) = @_; + if(!$n) { + $n = $p; + } + return sprintf("%s;%s;$desc", $p, $n); +} + +my $usesamehttpversion = 1; +my $disableheadersnotseen = 1; +my $shellcompatible = 1; # may not been windows command prompt compat +my $uselongoptions = 1; # instead of short +my $uselibcurl = 0; # --libcurl +my $usehttp = 0; + +while($ARGV[0]) { + if(($ARGV[0] eq "-h") || ($ARGV[0] eq "--help")) { + usage(); + } + elsif($ARGV[0] eq "-a") { + $disableheadersnotseen = 0; + shift @ARGV; + } + elsif($ARGV[0] eq "-d") { + $usedocs = 1; + shift @ARGV; + } + elsif($ARGV[0] eq "-H") { + $usehttp = 1; + shift @ARGV; + } + elsif($ARGV[0] eq "-i") { + $usesamehttpversion = 0; + shift @ARGV; + } + elsif($ARGV[0] eq "--libcurl") { + $uselibcurl = 1; + shift @ARGV; + } + elsif($ARGV[0] eq "-n") { + $usenotes = 1; + shift @ARGV; + } + elsif($ARGV[0] eq "-s") { + $uselongoptions = 0; + shift @ARGV; + } + elsif($ARGV[0] eq "-v") { + $useverbose = 1; + shift @ARGV; + } + else { + usage(); + } +} + + +my $state; # 0 is request-line, 1-headers, 2-body +my $line = 1; +while() { + my $l = $_; + # discard CRs completely + $l =~ s/\r//g; + if(!$state) { + chomp $l; + if($l =~ /([^ ]*) +(.*) +(HTTP\/.*)/) { + $method = $1; + $path = $2; + $http = $3; + } + else { + $error="bad request-line"; + last; + } + $state++; + } + elsif(1 == $state) { + chomp $l; + if($l =~ /([^:]*): *(.*)/) { + $header{lc($1)}=$2; + $exactcase{lc($1)}=$1; # to allow us to use it as read + } + elsif(length($l)<2) { + # body time + $state++; + } + else { + $error="illegal HTTP header on line $line"; + last; + } + } + elsif(2 == $state) { + push @body, $l; + } + $line++; +} + +if(!$header{'host'}) { + $error = "No Host: header makes it impossible to tell URL\n"; +} + + error: +if($error) { + print "Error: $error\n"; + exit; +} + +if($uselongoptions) { + $opt_data = "--data"; + $opt_request = "--request"; + $opt_head = "--head"; + $opt_header = "--header"; + $opt_user_agent = "--user-agent"; + $opt_cookie = "--cookie"; + $opt_verbose = "--verbose"; + $opt_form = "--form"; + $opt_user = "--user"; +} +else { + $opt_data = "-d"; + $opt_request = "-X"; + $opt_head = "-I"; + $opt_header = "-H"; + $opt_user_agent = "-A"; + $opt_cookie = "-b"; + $opt_verbose = "-v"; + $opt_form = "-F"; + $opt_user = "-u"; +} + +my $httpver=""; +my $disabledheaders=""; +my $addedheaders=""; + +if($header{"content-type"} =~ /^multipart\/form-data;/) { + # multipart formpost, this is special + my $type = $header{"content-type"}; + my $boundary = $type; + $boundary =~ s/.*boundary=(.*)/$1/; + my $inbound = $body[0]; + chomp $inbound; + # a body MUST start with dash-dash-boundary + if("--$boundary" ne $inbound) { + $error = "unexpected multipart format"; + goto error; + } + my $bline=1; + + my %fheader; + my $fstate = 0; + my @fbody; + while($body[$bline]) { + my $l = $body[$bline]; + if(0 == $fstate) { + # headers + chomp $l; + if($l =~ /([^:]*): *(.*)/) { + $fheader{lc($1)}=$2; + } + elsif(length($l)<2) { + # body time + $fstate++; + } + } + elsif($fstate) { + if($l =~ /^--$boundary/) { + # end of this part + my $cd = $fheader{'content-disposition'}; + if(!$cd) { + $error = "multi-part without Content-Disposition: header!"; + goto error; + } + # Content-Disposition: form-data; name="name" + # Content-Disposition: form-data; name="file"; filename="README.md" + if($cd =~ /^form-data; name=([^;]*)[;]? *(.*)/i) { + my ($n, $f)=($1, $2); + # name is with or without quotes + $n =~ s/\"//g; + if($f =~ /^filename=(.*)/) { + # filename is with or without quotes + $f = $1; + $f =~ s/\"//g; + } + if(!$multipart) { + push @docs, manpage("-F", $opt_form, "send a multipart formpost"); + } + if(!$f) { + my $fbody = join("", @fbody); + $fbody =~ s/[ \n\r]+\z//g; + $fbody =~ s/([\\\$\"\'\`])/\\$1/g; + $multipart .= "$opt_form $n=\"$fbody\" "; + @fbody=""; + } + else { + # file name was present + $multipart .= "$opt_form $n=\@$f "; + } + } + $fstate = 0; + %fheader = 0; + $bline++; + next; + } + push @fbody, $l; + } + $bline++; + } + if($body[$bline-1] !~ /^--$boundary--/) { + print STDERR "bad last line?"; + } + + $header{"content-type"} = ""; # blank it + $do_multipart = 1; + +} +elsif(length(join("", @body))) { + # TODO: escape the body + my $esc = join("", @body); + chomp $esc; # trim the final newline + if($shellcompatible) { + $esc =~ s/\n/ /g; # turn newlines into space! + $esc =~ s/\"/\\"/g; # escape double quotes + if(!$unixescaped) { + push @notes, "uses quotes suitable for *nix command lines"; + $unixescaped++; + } + } + $usebody= sprintf("--data-binary \"%s\" ", $esc); + push @docs, manpage("--data-binary", "", "send this string as a body with POST"); +} +if(uc($method) eq "HEAD") { + $usemethod = "$opt_head "; + push @docs, manpage("-I", $opt_head, "send a HEAD request"); +} +elsif(uc($method) eq "POST") { + if(!$usebody && !$do_multipart) { + $usebody= sprintf("$opt_data \"\" "); + push @docs, manpage("-d", $opt_data, "send this string as a body with POST"); + } +} +elsif(uc($method) eq "PUT") { + if(!$usebody) { + $usebody= sprintf("$opt_data \"\" "); + push @docs, manpage("-d", $opt_data, "send this string as a body with POST"); + } + $usebody .= "$opt_request PUT "; + push @docs, manpage("-X", $opt_request, "replace the request method with this string"); +} +elsif(uc($method) eq "OPTIONS") { + $usemethod .= "$opt_request OPTIONS "; + if($path !~ /^\//) { + # very special case + $requesttarget="--request-target \"$path\" "; + push @docs, manpage("--request-target", "", + "specify request target to use instead of using the URL's"); + $path = ""; + } +} +elsif(uc($method) ne "GET") { + $error = "unsupported HTTP method $method"; + goto error; +} + +if($usebody) { + # body is set, handle the content-type + if(!$header{"content-type"}) { + $disabledheaders .= "$opt_header Content-Type: "; + } + elsif(lc($header{"content-type"}) ne + "application/x-www-form-urlencoded") { + # custom + $ignore_contenttype = 1; + $addedheaders .= sprintf("$opt_header \"Content-Type: %s\" ", + $header{"content-type"}); + } + elsif((lc($header{"content-type"}) eq + "application/x-www-form-urlencoded") && !$do_multipart && + ($method eq "POST")) { + # default for normal POST + $ignore_contenttype = 1; + } +} + +if($usesamehttpversion) { + if(uc($http) eq "HTTP/1.1") { + $httpver = "--http1.1 "; + push @docs, manpage("--http1.1", "", "use HTTP protocol version 1.1"); + } + elsif(uc($http) eq "HTTP/2") { + $httpver = "--http2 "; + push @docs, manpage("--http2", "", "use HTTP protocol version 2"); + } + else { + $error = "unsupported HTTP version $http"; + goto error; + } +} +if($disableheadersnotseen) { + if(!$header{'accept'}) { + $disabledheaders .= "$opt_header Accept: "; + } + if(!$header{'user-agent'}) { + $disabledheaders .= "$opt_header User-Agent: "; + } +} + +if($do_multipart) { + if(!$header{lc("expect")}) { + # no expect header, disable it for us too since curl -F defaults to + # Expect: 100-continue + $disabledheaders .= "$opt_header Expect: "; + } +} + +# go through the headers alphabetically just to make the order fixed +foreach my $h (sort keys %header) { + if(lc($h) eq "host") { + # We use Host: for the URL creation + } + elsif((lc($h) eq "authorization") && + ($header{'authorization'} =~ /^Basic (.*)/)) { + my $decoded = decode_base64($1); + $addedheaders .= sprintf("%s \"%s\" ", $opt_user, $decoded); + push @docs, manpage("-u", $opt_user, "use this user and password for Basic auth"); + } + elsif(lc($h) eq "expect") { + # let curl do expect on its own + } + elsif(lc($h) eq "content-type" && + ($do_multipart || $ignore_contenttype)) { + # skip this for multipart + } + elsif(($h eq "accept-encoding") && + ($header{$h} =~ /gzip/)) { + push @docs, manpage("--compressed", "", "request a compressed response"); + $addedheaders .= "--compressed "; + } + elsif((lc($h) eq "accept") && + ($header{"accept"} eq "*/*")) { + # ignore if set to */* as that's a curl default + } + elsif(lc($h) eq "content-length") { + # we don't set custom size, just usebody + } + else { + $exact = $exactcase{$h}; + my $opt = sprintf("$opt_header \"%s: ", $exact); + if(lc($h) eq "user-agent") { + $opt = "$opt_user_agent \""; + push @docs, manpage("-A", $opt_user_agent, "use this custom User-Agent request header"); + } + elsif(lc($h) eq "cookie") { + $opt = "$opt_cookie \""; + push @docs, manpage("-b", $opt_cookie, "pass on this custom Cookie: request header"); + } + $addedheaders .= sprintf("%s%s\" ", $opt, $header{$h}); + } +} + +if($path =~ /[ &?]/) { + $url = sprintf "\"%s://%s%s\"", $usehttp ? "http" : "https", $header{'host'}, $path; +} +else { + $url = sprintf "%s://%s%s", $usehttp ? "http" : "https", $header{'host'}, $path; +} + +if($disabledheaders || $addedheaders) { + push @docs, manpage("-H", $opt_header, "add, replace or remove HTTP headers from the request"); +} + +if($useverbose) { + $useverbose = "$opt_verbose "; + push @docs, manpage("-v", $opt_verbose, "show verbose output"); +} + +# This adds the -x option to prevent a curl request to actually go out to any +# remote server +my $lib="--libcurl - -x localhost:0 " if($uselibcurl); +my $curlcmd = sprintf "curl ${useverbose}${usemethod}${httpver}${disabledheaders}${addedheaders}${usebody}${multipart}${requesttarget}${lib}${url}"; + +if($uselibcurl) { + # this actually runs curl which will fail to connect so ignore errors + open(C, "$curlcmd 2>/dev/null|"); + while() { + # skip CURLOPT_PROXY since that's only used to avoid network + if($_ !~ /CURLOPT_PROXY, /) { + print $_; + } + } + close(C); +} +else { + print "$curlcmd\n"; +} + +if($usenotes) { + print "---\n"; + foreach my $n (@notes) { + print "$n\n"; + } +} + +if($usedocs) { + print "---\n"; + foreach my $d (@docs) { + print "$d\n"; + } +} diff --git a/man/h2c.Rd b/man/h2c.Rd new file mode 100644 index 0000000..5bf1ea2 --- /dev/null +++ b/man/h2c.Rd @@ -0,0 +1,29 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/h2c.R +\name{h2c} +\alias{h2c} +\title{Convert an 'httr' call to 'curl' command line} +\usage{ +h2c(complete_httr_verb_call) +} +\arguments{ +\item{complete_httr_verb_call}{wrap an \code{httr} \code{VERB} call with this function +and it will return the text of a working \code{curl} command line} +} +\description{ +Convert an 'httr' call to 'curl' command line +} +\examples{ +\dontrun{ +h2c( + httr::GET( + url = "https://rud.is/", + httr::user_agent(splashr::ua_apple_tv), + query = list( + a = "b", + c = 1 + ) + ) + ) +} +} diff --git a/man/httr2curl.Rd b/man/httr2curl.Rd index 0618673..39a518a 100644 --- a/man/httr2curl.Rd +++ b/man/httr2curl.Rd @@ -4,9 +4,11 @@ \name{httr2curl} \alias{httr2curl} \alias{httr2curl-package} -\title{...} +\title{Turn 'httr' Web Requests Into 'curl' Command Line Calls} \description{ -A good description goes here otherwise CRAN checks fail. +When provided with an 'httr' "VERB" call, the request will be made +and 'libcurl' headers will be captured and post-processed to provide a working +'curl' command line call. } \seealso{ Useful links: diff --git a/man/pipe.Rd b/man/pipe.Rd new file mode 100644 index 0000000..0eec752 --- /dev/null +++ b/man/pipe.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/utils-pipe.R +\name{\%>\%} +\alias{\%>\%} +\title{Pipe operator} +\usage{ +lhs \%>\% rhs +} +\description{ +See \code{magrittr::\link[magrittr:pipe]{\%>\%}} for details. +} +\keyword{internal}