|
1 | 1 | # sqlite-http-cache |
2 | | -SQLite Extension to cache HTTP requests |
3 | 2 |
|
4 | | -and an [SQLite http proxy cache](#sqlite-proxy-cache). |
| 3 | +This repository contains tools to integrate with the [httpcache SQLite Extension](https://github.com/litesql/httpcache). |
5 | 4 |
|
6 | | -## Installation |
7 | 5 |
|
8 | | -Download **httpcache** extension from the [releases page](https://github.com/walterwanderley/sqlite-http-cache/releases). |
| 6 | +## SQLite HTTP Proxy Cache |
9 | 7 |
|
10 | | -### Compiling from source |
11 | | - |
12 | | -- [Go 1.24+](https://go.dev) is required. |
| 8 | +The sqlite-http-proxy is an HTTP proxy cache that can store data in multiple sqlite databases and query concurrently to get the faster response. The cache implements [RFC9111](https://www.rfc-editor.org/rfc/rfc9111.html) (except for the Vary header). |
13 | 9 |
|
14 | | -```sh |
15 | | -go build -ldflags="-s -w" -buildmode=c-shared -o httpcache.so |
16 | | -``` |
| 10 | +1. Installation: |
17 | 11 |
|
18 | | -## Basic usage |
| 12 | +Download sqlite-http-proxy from the [releases page](https://github.com/walterwanderley/sqlite-http-cache/releases), or install from source: |
19 | 13 |
|
20 | 14 | ```sh |
21 | | -sqlite3 |
22 | | - |
23 | | -# Load the extension |
24 | | -.load ./httpcache |
25 | | - |
26 | | -# Insert URL into the temp.http_request virtual table to trigger the HTTP Request |
27 | | -INSERT INTO temp.http_request VALUES('https://swapi.tech/api/films/1'); |
28 | | - |
29 | | -# Set output mode (optional) |
30 | | -.mode qbox |
31 | | - |
32 | | -# Fetch data from http_response table (created by the extension) |
33 | | -SELECT JSON_EXTRACT(body, '$.result.properties.title') AS title, |
34 | | - JSON_EXTRACT(body, '$.result.properties.release_date') AS release_date |
35 | | - FROM http_response; |
36 | | -┌──────────────┬──────────────┐ |
37 | | -│ title │ release_date │ |
38 | | -├──────────────┼──────────────┤ |
39 | | -│ 'A New Hope' │ '1977-05-25' │ |
40 | | -└──────────────┴──────────────┘ |
41 | | - |
42 | | -# Use cache_age, cache_lifetime or cache_expired function to check cache validity based on RFC9111 |
43 | | -SELECT url, cache_age(header, request_time, response_time) AS age, |
44 | | -cache_lifetime(header, response_time) AS lifetime, |
45 | | -cache_expired(header, request_time, response_time, false) AS expired, |
46 | | -cache_expired_ttl(header, request_time, response_time, false, 3600) AS expiredTTLFallback |
47 | | -FROM http_response; |
48 | | -┌──────────────────────────────────┬─────┬──────────┬─────────┬────────────────────┐ |
49 | | -│ url │ age │ lifetime │ expired │ expiredTTLFallback │ |
50 | | -├──────────────────────────────────┼─────┼──────────┼─────────┼────────────────────┤ |
51 | | -│ 'https://swapi.tech/api/films/1' │ 37 │ 0 │ 1 │ 0 │ |
52 | | -└──────────────────────────────────┴─────┴──────────┴─────────┴────────────────────┘ |
53 | | -``` |
54 | | - |
55 | | -All HTTP responses are stored in tables using the following schema: |
56 | | - |
57 | | -```sql |
58 | | -TABLE http_response( |
59 | | - url TEXT PRIMARY KEY, |
60 | | - status INTEGER, |
61 | | - body BLOB, |
62 | | - header JSONB, |
63 | | - request_time DATETIME, |
64 | | - response_time DATETIME |
65 | | -) |
| 15 | +go install github.com/walterwanderley/cmd/sqlite-http-proxy@latest |
66 | 16 | ``` |
67 | 17 |
|
68 | | -If you want to customize the persisted data, just create an another table and create triggers for "http_response". |
69 | | - |
70 | | -```sql |
71 | | -CREATE TABLE movies( |
72 | | - ID INTEGER PRIMARY KEY AUTOINCREMENT, |
73 | | - title TEXT, |
74 | | - release_date DATETIME |
75 | | -); |
76 | | - |
77 | | -CREATE TRIGGER insert_http_response |
78 | | -AFTER INSERT ON http_response |
79 | | -BEGIN |
80 | | - INSERT INTO movies(title, release_date) |
81 | | - VALUES( |
82 | | - JSON_EXTRACT(NEW.body, '$.result.properties.title'), |
83 | | - JSON_EXTRACT(NEW.body, '$.result.properties.release_date') |
84 | | - ); |
85 | | -END; |
86 | | - |
87 | | -INSERT INTO temp.http_request VALUES('https://swapi.tech/api/films/2'); |
88 | | - |
89 | | -SELECT * FROM movies; |
90 | | -┌────┬───────────────────────────┬──────────────┐ |
91 | | -│ ID │ title │ release_date │ |
92 | | -├────┼───────────────────────────┼──────────────┤ |
93 | | -│ 1 │ 'The Empire Strikes Back' │ '1980-05-17' │ |
94 | | -└────┴───────────────────────────┴──────────────┘ |
| 18 | +2. Executing: |
95 | 19 |
|
| 20 | +```sh |
| 21 | +sqlite-http-proxy --port 9090 --response-table http_response proxy1.db proxy2.db proxy3.db |
96 | 22 | ``` |
97 | 23 |
|
98 | | -## Configuring |
99 | | - |
100 | | -You can configure the behaviour by passing parameters to a VIRTUAL TABLE. |
101 | | - |
102 | | -| Param | Description | Default | |
103 | | -|-------|-------------|---------| |
104 | | -| timeout | Timeout in milliseconds | 0 | |
105 | | -| insecure | Insecure skip TLS validation | false | |
106 | | -| status_code | Comma-separated list of HTTP status code to persist. Use empty to persist all status | 200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501 | |
107 | | -| response_table | Database table used to store response data | http_response | |
108 | | -| oauth2_client_id | Oauth2 Client ID | | |
109 | | -| oauth2_client_secret | Oauth2 Client Secret | | |
110 | | -| oauth2_token_url | Oauth2 Token URL (Client Credentials Flow) | | |
111 | | -| cert_file | Mutual TLS: path to certificate file | | |
112 | | -| cert_key_file | Mutual TLS: path to certificate key file | | |
113 | | -| ca_file | Path to CA certificate file | | |
114 | | - |
115 | | -**Any other parameter will be included as an HTTP header in the request** |
116 | | - |
117 | | -You can use environment variables in parameter values. For example: |
| 24 | +3. Testing: |
118 | 25 |
|
119 | | -```sql |
120 | | -CREATE VIRTUAL TABLE temp.custom_request USING http_request(authorization='Bearer ${API_TOKEN}'); |
| 26 | +```sh |
| 27 | +time curl -x http://127.0.0.1:9090 http://swapi.tech/api/films/1 |
| 28 | +time curl -x http://127.0.0.1:9090 http://swapi.tech/api/films/1 |
121 | 29 | ``` |
122 | 30 |
|
123 | | -### Examples |
| 31 | +### Proxing HTTPS Requests |
124 | 32 |
|
125 | | -#### Customizing the request |
| 33 | +To proxy https requests you need to pass CA Certificate and CA Certificate key to the sqlite-http-proxy. |
126 | 34 |
|
127 | 35 | ```sh |
128 | | -# Create a Virtual Table to customize options |
129 | | -CREATE VIRTUAL TABLE temp.custom_request USING http_request(insecure=true, timeout=10000, accept=application/json, authorization='Bearer ${API_TOKEN}', response_table=films); |
130 | | - |
131 | | -# Insert URL into the Virtual Table to trigger the HTTP Request |
132 | | -INSERT INTO temp.custom_request VALUES('https://swapi.tech/api/films/2'); |
133 | | - |
134 | | -# Query the response table |
135 | | -SELECT JSON_EXTRACT(body, '$.result.properties.title') AS title, |
136 | | - JSON_EXTRACT(body, '$.result.properties.release_date') AS release_date |
137 | | - FROM films; |
| 36 | +sqlite-http-proxy --ca-cert=/path/to/ca.crt --ca-cert-key=/path/to/ca.key proxyN.db |
138 | 37 | ``` |
139 | 38 |
|
140 | | -#### Configuring Oauth2 Client Credentials |
| 39 | +Use the command line flag --help for more info. |
141 | 40 |
|
142 | 41 | ```sh |
143 | | -CREATE VIRTUAL TABLE temp.oauth2_request USING http_request(oauth2_client_id=${CLIENT_ID}, oauth2_client_secret=${CLIENT_SECRET}, oauth2_token_url='https://my-token-url'); |
144 | | - |
145 | | -INSERT INTO temp.oauth2_request VALUES('https://swapi.tech/api/films/3'); |
146 | | - |
147 | | -SELECT JSON_EXTRACT(body, '$.result.properties.title') AS title, |
148 | | - JSON_EXTRACT(body, '$.result.properties.release_date') AS release_date |
149 | | - FROM http_response; |
| 42 | +sqlite-http-proxy --help |
150 | 43 | ``` |
151 | 44 |
|
152 | 45 | ## Refresh data |
@@ -178,110 +71,4 @@ INSERT INTO temp.http_request |
178 | 71 | SELECT url FROM http_response |
179 | 72 | WHERE unixepoch() - unixepoch(response_time) > :ttl ; |
180 | 73 | ``` |
181 | | -*ttl is Time to Live in seconds* |
182 | | - |
183 | | -## SQLite Proxy Cache |
184 | | - |
185 | | -The sqlite-http-proxy is an HTTP proxy cache that can store data in multiple sqlite databases and query concurrently to get the faster response. The cache implements [RFC9111](https://www.rfc-editor.org/rfc/rfc9111.html) (except for the Vary header). |
186 | | - |
187 | | -1. Installation: |
188 | | - |
189 | | -Download sqlite-http-proxy from the [releases page](https://github.com/walterwanderley/sqlite-http-cache/releases), or install from source: |
190 | | - |
191 | | -```sh |
192 | | -go install github.com/walterwanderley/cmd/sqlite-http-proxy@latest |
193 | | -``` |
194 | | - |
195 | | -2. Executing: |
196 | | - |
197 | | -```sh |
198 | | -sqlite-http-proxy --port 9090 --response-table http_response proxy1.db proxy2.db proxy3.db |
199 | | -``` |
200 | | - |
201 | | -3. Testing: |
202 | | - |
203 | | -```sh |
204 | | -time curl -x http://127.0.0.1:9090 http://swapi.tech/api/films/1 |
205 | | -time curl -x http://127.0.0.1:9090 http://swapi.tech/api/films/1 |
206 | | -``` |
207 | | - |
208 | | -### Proxing HTTPS Requests |
209 | | - |
210 | | -To proxy https requests you need to pass CA Certificate and CA Certificate key to the sqlite-http-proxy. |
211 | | - |
212 | | -```sh |
213 | | -sqlite-http-proxy --ca-cert=/path/to/ca.crt --ca-cert-key=/path/to/ca.key proxyN.db |
214 | | -``` |
215 | | - |
216 | | -Use the command line flag --help for more info. |
217 | | - |
218 | | -```sh |
219 | | -sqlite-http-proxy --help |
220 | | -``` |
221 | | - |
222 | | -## Go HTTP Client |
223 | | - |
224 | | -This repository has an http.Transport implementation to use sqlite as cache of HTTP requests from golang. |
225 | | - |
226 | | -```go |
227 | | -package main |
228 | | - |
229 | | -import ( |
230 | | - "context" |
231 | | - "database/sql" |
232 | | - "log/slog" |
233 | | - "time" |
234 | | - |
235 | | - _ "github.com/mattn/go-sqlite3" |
236 | | - dbcache "github.com/walterwanderley/sqlite-http-cache/db" |
237 | | - httpcache "github.com/walterwanderley/sqlite-http-cache/http" |
238 | | -) |
239 | | - |
240 | | -func main() { |
241 | | - |
242 | | - db, err := sql.Open("sqlite3", "file:example.db") |
243 | | - if err != nil { |
244 | | - panic(err) |
245 | | - } |
246 | | - defer db.Close() |
247 | | - |
248 | | - tablesNames := []string{"example"} |
249 | | - |
250 | | - err = dbcache.CreateResponseTables(db, tablesNames...) |
251 | | - if err != nil { |
252 | | - panic(err) |
253 | | - } |
254 | | - |
255 | | - config := httpcache.Config{ |
256 | | - DB: db, |
257 | | - Tables: tablesNames, |
258 | | - ReadOnly: false, |
259 | | - TTL: 30 * time.Second, |
260 | | - RFC9111: false, |
261 | | - CleanupInterval: 0, |
262 | | - } |
263 | | - |
264 | | - client, err := config.Client(context.Background()) |
265 | | - if err != nil { |
266 | | - panic(err) |
267 | | - } |
268 | | - |
269 | | - start := time.Now() |
270 | | - resp, err := client.Get("http://swapi.tech/api/films/1") |
271 | | - if err != nil { |
272 | | - panic(err) |
273 | | - } |
274 | | - defer resp.Body.Close() |
275 | | - |
276 | | - slog.Info("first request", "duration", time.Since(start)) |
277 | | - |
278 | | - start = time.Now() |
279 | | - resp, err = client.Get("http://swapi.tech/api/films/1") |
280 | | - if err != nil { |
281 | | - panic(err) |
282 | | - } |
283 | | - defer resp.Body.Close() |
284 | | - |
285 | | - slog.Info("second request", "duration", time.Since(start)) |
286 | | -} |
287 | | -``` |
| 74 | +*ttl is Time to Live in seconds* |
0 commit comments