Repository navigation
Expand file tree
/
Copy pathcollection.rs
More file actions
190 lines (175 loc) · 8.44 KB
/
Copy pathcollection.rs
File metadata and controls
190 lines (175 loc) · 8.44 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
//! The collection of a target: after a switch, delete the files that only the releases that the
//! target no longer serves name, and the files under `objects/` and `releases/` that no release
//! names. A key of an older publisher, a pointer and `release.json` always stay.
use std::collections::{BTreeMap, BTreeSet};
use crate::release::{self, Release, POINTERS};
use crate::store::Store;
use crate::target::{Listed, Target};
/// How long the files of a release stay after it stops serving, in seconds: longer than a client
/// needs to fetch the files that a pointer it just read names. 10 minutes, chosen by the owner;
/// change it here.
pub const GRACE: u64 = 600;
/// The folders where a collection deletes each key that no kept release names. Elsewhere it deletes
/// only the temporary files of a stopped write.
const COLLECTED: [&str; 2] = ["objects/", "releases/"];
/// What a collection did.
#[derive(Debug, Default)]
pub struct Collected {
/// The releases whose files and manifest it deleted.
pub releases: Vec<String>,
/// The releases that `release.json` does not name, which stay for now.
pub deferred: Vec<String>,
/// The keys that it deleted, also of files that no release names, and their size.
pub keys: u64,
pub bytes: u64,
/// Why it stopped before the end: a pointer that is not of the serving release, a manifest that
/// it cannot read, or an error. `null` when it finished.
pub stopped: Option<String>,
}
impl Collected {
/// The collection in one line.
pub fn text(&self) -> String {
if let (Some(stopped), 0) = (&self.stopped, self.keys) {
return format!("collected nothing: {stopped}");
}
let mut text = format!("collected {} releases, {} keys, {} bytes", self.releases.len(), self.keys, self.bytes);
if !self.deferred.is_empty() {
text += &format!("; {} releases stay until a later collection", self.deferred.len());
}
if let Some(stopped) = &self.stopped {
text += &format!("; then it stopped: {stopped}");
}
text
}
}
/// How many keys a collection deletes in one request, and between two checks that the run still
/// holds the lock: the most that one S3 `DeleteObjects` request takes. Tests take 2, so that a
/// phase has more than one batch.
const BATCH: usize = if cfg!(test) { 2 } else { crate::r2::s3::DELETES };
/// Collect `target` at the time `now`, by the clock of the target, with `store` as the cache of the
/// lists of its releases. A release that `release.json`
/// does not name stopped serving at or before `served_since`, when the release that served before
/// the switch began to serve; or at an unknown time when it is `None`. The first collection
/// [`GRACE`] seconds or more after that time deletes it. The run holds the lock of the target, so no
/// other run writes a file meanwhile; `holds` checks that before each [`BATCH`] of deletes.
pub fn collect(
(target, store): (&Target, &Store),
served_since: Option<&str>,
now: u64,
holds: &dyn Fn() -> Result<(), String>,
) -> Collected {
let mut collected = Collected::default();
if let Err(e) = pass((target, store), served_since, now, holds, &mut collected) {
collected.stopped = Some(e);
}
collected
}
fn pass(
(target, store): (&Target, &Store),
served_since: Option<&str>,
now: u64,
holds: &dyn Fn() -> Result<(), String>,
collected: &mut Collected,
) -> Result<(), String> {
let Some((serving, _)) = release::serving(target)? else { return Ok(()) };
let current = Release::read(target, &serving.serving.id)?;
if let Some(key) = release::drift(target, ¤t)?.into_iter().next() {
return Err(format!("{key} is not of the serving release"));
}
let mut kept = vec![current];
if let Some(rollback) = &serving.rollback {
kept.push(Release::read(target, &rollback.id)?);
}
let listed: BTreeMap<String, Listed> =
target.list()?.into_iter().map(|listed| (listed.key.clone(), listed)).collect();
let expired = served_since.and_then(crate::date::seconds).is_some_and(|since| since.saturating_add(GRACE) <= now);
let named: BTreeSet<&str> =
std::iter::once(&serving.serving).chain(&serving.rollback).map(|role| role.id.as_str()).collect();
let unknown = |e: String| format!("the files of a release are not known: {e}");
let mut dropped = Vec::new();
for (id, manifest) in listed.iter().filter_map(|(key, listed)| Some((manifest_id(key)?, listed))) {
if named.contains(id) {
continue;
}
let release = Release::read(target, id).map_err(unknown)?;
if expired {
dropped.push((manifest.clone(), release));
} else {
collected.deferred.push(id.into());
kept.push(release);
}
}
let mut keep = BTreeSet::new();
let store = store.reading(target);
for release in &kept {
let files = release.load(release::in_store(&store)).map_err(unknown)?;
keep.extend(files.files().into_iter().map(|file| file.key));
}
// The files first, then the manifests, then their lists: a collection that stops resumes from
// the manifests that remain, and loads them.
let lists: BTreeSet<String> =
dropped.iter().flat_map(|(_, release)| release.lists()).map(|list| list.key).collect();
let files = listed.values().filter(|listed| {
manifest_id(&listed.key).is_none()
&& !keep.contains(&listed.key)
&& !lists.contains(&listed.key)
&& (collectable(&listed.key) || temporary(&listed.key))
});
let manifests = dropped.into_iter().map(|(manifest, _)| manifest);
let lists = lists.iter().filter(|key| !keep.contains(*key)).filter_map(|key| listed.get(key));
let phases: [Vec<Listed>; 3] = [files.cloned().collect(), manifests.collect(), lists.cloned().collect()];
for keys in phases {
for batch in keys.chunks(BATCH) {
holds()?;
let stays = target.remove_many(&batch.iter().map(|listed| listed.key.clone()).collect::<Vec<_>>())?;
deleted(collected, batch, &stays)?;
}
}
Ok(())
}
/// Count the keys of `batch` but those that `stays` names with why, and stop when a key stays.
fn deleted(collected: &mut Collected, batch: &[Listed], stays: &[(String, String)]) -> Result<(), String> {
for listed in batch.iter().filter(|listed| !stays.iter().any(|(key, _)| *key == listed.key)) {
(collected.keys, collected.bytes) = (collected.keys + 1, collected.bytes + listed.size);
collected.releases.extend(manifest_id(&listed.key).map(String::from));
}
match stays.first() {
None => Ok(()),
Some((key, why)) => Err(format!("{} keys stay, such as {key}: {why}", stays.len())),
}
}
/// The id of the manifest `releases/<id>.json`.
fn manifest_id(key: &str) -> Option<&str> {
let id = key.strip_prefix("releases/")?.strip_suffix(".json")?;
release::is_sha256(id).then_some(id)
}
/// Whether a collection may delete `key`: a key in one of [`COLLECTED`].
fn collectable(key: &str) -> bool {
COLLECTED.iter().any(|folder| key.starts_with(folder))
}
/// Whether `key` is a part file of an upload or a temporary file of a write that stopped: in the
/// root of the target, or a sibling of a pointer that names the pointer.
fn temporary(key: &str) -> bool {
let (folder, name) = key.rsplit_once('/').unwrap_or(("", key));
let hidden = name.starts_with('.') && (name.ends_with(".part") || name.ends_with(".tmp"));
let beside = |pointer: &str| {
let (pointer_folder, file) = pointer.rsplit_once('/').unwrap_or(("", pointer));
pointer_folder == folder && name.starts_with(&format!(".{file}."))
};
hidden && (folder.is_empty() || POINTERS.iter().any(|pointer| beside(pointer)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_batch_counts_the_keys_that_went_also_when_a_key_stays() {
let manifest = format!("releases/{}.json", "a".repeat(64));
let batch = [("objects/a", 1), ("objects/b", 2), (manifest.as_str(), 4)]
.map(|(key, size)| Listed { key: key.into(), size });
let mut collected = Collected::default();
let stays = [("objects/b".to_string(), "InternalError".to_string())];
let stopped = deleted(&mut collected, &batch, &stays).unwrap_err();
assert_eq!(stopped, "1 keys stay, such as objects/b: InternalError");
assert_eq!((collected.keys, collected.bytes, collected.releases), (2, 5, vec!["a".repeat(64)]));
}
}