aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorpukkamustard <pukkamustard@posteo.net>2021-03-27 09:05:29 +0100
committerpukkamustard <pukkamustard@posteo.net>2021-03-27 09:05:29 +0100
commit5f90212bb5d5b225741c991f583ca3a4131c9d88 (patch)
treebe3bd7a6e943b7cd09ecd12ec66a19b386e9283b
parent0879e08c84ade48616edcb2ebef62f9f8f5b6982 (diff)
(eris ipfs): Add interface to IPFS for storing and retrieving blocks.
-rw-r--r--Makefile.am1
-rw-r--r--README.org32
-rw-r--r--eris/ipfs.scm163
-rw-r--r--guix.scm3
-rw-r--r--hall.scm1
5 files changed, 197 insertions, 3 deletions
diff --git a/Makefile.am b/Makefile.am
index 7db238d..43be38e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -38,6 +38,7 @@ SOURCES = eris.scm \
eris/encode.scm \
eris/decode.scm \
eris/read-capability.scm \
+ eris/ipfs.scm \
eris/utils/padding.scm \
eris/utils/base32.scm \
eris/utils/rbytevector.scm
diff --git a/README.org b/README.org
index d7d31f1..a9524ea 100644
--- a/README.org
+++ b/README.org
@@ -18,7 +18,6 @@ ERIS describes a scheme for splitting up content into uniformly sized encrypted
ERIS does not define storage and transport layer. It relies on a block storage that allows block to be stored and referenced via the hash code of the block itself (content-addressing).
-See the [[./examples/ipfs.org][IPFS example]] for how IPFS can be used as storage and transport layer.
** Encoding
@@ -121,7 +120,36 @@ By default ~eris-encode~ returns an alist of references and blocks. Under the ho
The string "Hello world!" is encoded in a single block.
-See the [[./examples/ipfs.org][IPFS example]] for more details and on how IPFS can be used as a storage/transport layer.
+*** IPFS
+
+IPFS can be used to store and transport block of ERIS encoded content. The ~(eris ipfs)~ module provides the necessary interface to the IPFS HTTP API.
+
+After starting the IPFS daemon (with ~ipfs daemon~), content can be encoded with ERIS and stored on IPFS:
+
+#+BEGIN_SRC scheme :exports both
+(use-modules (eris)
+ (eris ipfs))
+
+
+(eris-encode "Hello world!"
+ #:block-reducer (ipfs-block-reducer))
+#+END_SRC
+
+#+RESULTS:
+: #<<read-capability> block-size: 1024 level: 0 root-reference: #vu8(63 254 3 75 10 5 103 7 208 238 26 103 0 126 217 124 236 105 205 75 136 116 101 176 189 63 118 210 40 169 217 105) root-key: #vu8(19 230 50 100 252 2 228 177 6 63 67 102 128 217 107 51 19 246 52 165 17 175 247 177 68 0 59 165 100 99 44 219)>
+: done
+
+#+BEGIN_SRC scheme :exports both
+(utf8->string
+ (eris-decode->bytevector
+ "urn:erisx2:AAAD77QDJMFAKZYH2DXBUZYAP3MXZ3DJZVFYQ5DFWC6T65WSFCU5S2IT4YZGJ7AC4SYQMP2DM2ANS2ZTCP3DJJIRV733CRAAHOSWIYZM3M"
+ ipfs-block-ref))
+#+END_SRC
+
+#+RESULTS:
+: Hello world!
+
+See the [[./examples/ipfs.org][IPFS write-up]] for details on how IPFS can be used as storage and transport layer.
** Block size and convergence secret
diff --git a/eris/ipfs.scm b/eris/ipfs.scm
new file mode 100644
index 0000000..498dc89
--- /dev/null
+++ b/eris/ipfs.scm
@@ -0,0 +1,163 @@
+; SPDX-FileCopyrightText: 2021 pukkamustard <pukkamustard@posteo.net>
+; Copyright © 2018 Ludovic Courtès <ludo@gnu.org>
+;
+; SPDX-License-Identifier: GPL-3.0-or-later
+
+;; This module provides an interface to the IPFS daemons HTTP API for storing
+;; and retrieving blocks. This can be used to store blocks of ERIS encoded content.
+;;
+;; See also the IPFS API documentation:
+;; https://docs.ipfs.io/reference/http/api/#api-v0-block-put
+;;
+;; Parts are taken from guix/ipfs.scm in the wip-ipfs-substitutes branch of the
+;; GNU Guix repository.
+
+(define-module (eris ipfs)
+
+ #:use-module (eris utils base32)
+
+ #:use-module (sodium generichash)
+
+ #:use-module (json)
+
+ #:use-module (web uri)
+ #:use-module (web client)
+ #:use-module (web response)
+
+ #:use-module (srfi srfi-71)
+
+ #:use-module (rnrs io ports)
+ #:use-module (rnrs bytevectors)
+
+ #:export (ipfs-block-reducer
+ ipfs-block-ref))
+
+
+;; CID encoding
+
+;; Multicodec codes (https://github.com/multiformats/multicodec/blob/master/table.csv)
+(define multicodec-raw-code #x55)
+(define multicodec-blake2b-256-code #xb220)
+
+(define (blake2b-256->binary-cid hash)
+ "Encode a Blake2b-256 hash as binary CID"
+ (call-with-values
+ (lambda () (open-bytevector-output-port))
+ (lambda (port get-bytevector)
+ ;; CID version
+ (put-u8 port 1)
+ ;; multicoded content-type
+ (put-u8 port multicodec-raw-code)
+ ;; set multihash to blake2b-256. This is the manually encoded varint of 0xb220
+ (put-u8 port 160) (put-u8 port 228) (put-u8 port 2)
+ ;; set hash lenght
+ (put-u8 port 32)
+ ;; and finally the hash itself
+ (put-bytevector port hash)
+
+ ;; finalize and get the bytevector
+ (get-bytevector))))
+
+(define (binary-cid->cid bcid)
+ "Encode a binary CID as Base32 encoded CID"
+ ;; 'b' is the multibsae code for base32
+ (string-append "b"
+ ;; the IPFS daemon uses lower-case, so to be consistent we also.
+ (string-downcase
+ ;; base32 encode the binary cid
+ (base32-encode bcid))))
+
+(define blake2b-256->cid
+ (compose binary-cid->cid blake2b-256->binary-cid))
+
+
+;; IPFS API
+
+(define %default-ipfs-base-url
+ ;; URL of the IPFS gateway.
+ "http://localhost:5001")
+
+(define* (call url decode #:optional (method http-post)
+ #:key body (false-if-404? #t) (headers '()))
+ "Invoke the endpoint at URL using METHOD. Decode the resulting JSON body
+using DECODE, a one-argument procedure that takes an input port; when DECODE
+is false, return the input port. When FALSE-IF-404? is true, return #f upon
+404 responses."
+ (let* ((response port
+ (method url #:streaming? #t
+ #:body body
+
+ ;; Always pass "Connection: close".
+ #:keep-alive? #f
+ #:headers `((connection close)
+ ,@headers))))
+ (cond ((= 200 (response-code response))
+ (if decode
+ (let ((result (decode port)))
+ (close-port port)
+ result)
+ port))
+ ((and false-if-404?
+ (= 404 (response-code response)))
+ (close-port port)
+ #f)
+ (else
+ (close-port port)
+ (throw 'ipfs-error url response)))))
+
+(define %multipart-boundary
+ ;; XXX: We might want to find a more reliable boundary.
+ (string-append (make-string 24 #\-) "2698127afd7425a6"))
+
+(define (bytevector->form-data bv port)
+ "Write to PORT a 'multipart/form-data' representation of BV."
+ (display (string-append "--" %multipart-boundary "\r\n"
+ "Content-Disposition: form-data\r\n"
+ "Content-Type: application/octet-stream\r\n\r\n")
+ port)
+ (put-bytevector port bv)
+ (display (string-append "\r\n--" %multipart-boundary "--\r\n")
+ port))
+
+(define* (ipfs-block-put bv #:key (ipfs-base-url %default-ipfs-base-url))
+ "Store a block on IPFS and return the CID of the block"
+ (call (string-append ipfs-base-url
+ "/api/v0/block/put"
+ "?format=raw&mhtype=blake2b-256")
+ (lambda (port) (assoc-ref (json->scm port) "Key"))
+ #:headers `((content-type
+ . (multipart/form-data
+ (boundary . ,%multipart-boundary))))
+ #:body (call-with-bytevector-output-port
+ (lambda (port) (bytevector->form-data bv port)))))
+
+(define* (ipfs-block-get cid #:key (ipfs-base-url %default-ipfs-base-url))
+ "Get a block from IPFS via the HTTP API"
+ (call (string-append ipfs-base-url
+ "/api/v0/block/get"
+ "?arg=" cid)
+ get-bytevector-all))
+
+;; ERIS block reducer
+
+(define* (ipfs-block-reducer #:key (ipfs-base-url %default-ipfs-base-url))
+ "Returns a block reducer that stores block on IPFS"
+ (case-lambda
+ ;; initialization. Nothing to do here. In an improved implementation we
+ ;; might create a single HTTP connection and reuse it for all blocks.
+ (() '())
+
+ ;; Completion. Again, nothing to do.
+ ((_) 'done)
+
+ ;; store a block
+ ((_ ref-block)
+ ;; ref-block is a pair consisting of the reference to the block and the block itself.
+ (ipfs-block-put (cdr ref-block)
+ #:ipfs-base-url ipfs-base-url))))
+
+(define* (ipfs-block-ref ref #:key (ipfs-base-url %default-ipfs-base-url))
+ "Dereference a block from IPFS"
+ (ipfs-block-get (blake2b-256->cid ref)
+ #:ipfs-base-url ipfs-base-url))
+
diff --git a/guix.scm b/guix.scm
index 47dafb6..9b54b9d 100644
--- a/guix.scm
+++ b/guix.scm
@@ -65,7 +65,8 @@ tools.")
("asciidoctor" ,ruby-asciidoctor)))
(inputs `(("guile" ,guile-3.0)))
(propagated-inputs
- `(("guile-sodium" ,guile-sodium)))
+ `(("guile-sodium" ,guile-sodium)
+ ("guile-json" ,guile-json)))
(synopsis
"Guile implementation of Encoding for Robust Immutable Storage (ERIS)")
(description
diff --git a/hall.scm b/hall.scm
index 2020d55..ec911f1 100644
--- a/hall.scm
+++ b/hall.scm
@@ -22,6 +22,7 @@
(directory "eris" ((scheme-file "encode")
(scheme-file "decode")
(scheme-file "read-capability")
+ (scheme-file "ipfs")
(directory
"utils"
((scheme-file "padding")