feat: implement sample and analysis store
This commit is contained in:
parent
170ca32f58
commit
ce641749d2
13 changed files with 3086 additions and 3 deletions
553
src/store/analysis.rs
Normal file
553
src/store/analysis.rs
Normal file
|
|
@ -0,0 +1,553 @@
|
|||
use std::{
|
||||
collections::BTreeMap,
|
||||
fs::{self, File},
|
||||
io::Read as _,
|
||||
path::{Component, Path, PathBuf},
|
||||
str::FromStr as _,
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use crate::{domain::Digest, operation::AnalysisProfile};
|
||||
|
||||
use super::{
|
||||
AnalysisLock, AnalysisStore, StoreError, StoreErrorKind,
|
||||
filesystem::{create_private_directory, random_id, sync_directory},
|
||||
};
|
||||
|
||||
/// Complete immutable Analysis manifest.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AnalysisManifest {
|
||||
/// Manifest format version, exactly 1.
|
||||
pub manifest_version: u32,
|
||||
/// Completion marker, exactly `complete`.
|
||||
pub state: String,
|
||||
/// Staged Sample identity and byte length.
|
||||
pub sample: ManifestSample,
|
||||
/// Analysis Profile identity and compatibility facts.
|
||||
pub analysis_profile: AnalysisProfileManifest,
|
||||
/// Tool and invocation that completed the Analysis.
|
||||
pub created_by: CreatedBy,
|
||||
/// Fixed project name, path, and complete file inventory.
|
||||
pub project: AnalysisProject,
|
||||
}
|
||||
|
||||
/// Sample facts recorded in an Analysis manifest.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ManifestSample {
|
||||
/// SHA-256 Sample identity.
|
||||
pub sha256: Digest,
|
||||
/// Exact Sample byte size.
|
||||
pub size_bytes: u64,
|
||||
}
|
||||
|
||||
/// Compatibility facts recorded alongside the Profile digest.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AnalysisProfileManifest {
|
||||
/// Canonical Analysis Profile digest.
|
||||
pub sha256: Digest,
|
||||
/// Exact Ghidra version.
|
||||
pub ghidra_version: String,
|
||||
/// Rust/Java adapter protocol version.
|
||||
pub adapter_protocol_version: u32,
|
||||
/// Fully resolved Target Specification facts.
|
||||
pub target: BTreeMap<String, Value>,
|
||||
/// Digest of the fully resolved analyzer-option document.
|
||||
pub analyzer_options_sha256: Digest,
|
||||
}
|
||||
|
||||
/// Creator provenance for one immutable Analysis.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CreatedBy {
|
||||
/// CLI version.
|
||||
pub ghidr_version: String,
|
||||
/// Canonical random invocation ID.
|
||||
pub invocation_id: String,
|
||||
/// RFC3339 UTC completion timestamp supplied by orchestration.
|
||||
pub completed_at: String,
|
||||
}
|
||||
|
||||
/// Fixed project metadata and complete inventory.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AnalysisProject {
|
||||
/// Fixed project basename, exactly `analysis`.
|
||||
pub name: String,
|
||||
/// Fixed relative project directory, exactly `project`.
|
||||
pub path: String,
|
||||
/// Every regular file in bytewise relative-path order.
|
||||
pub files: Vec<InventoryFile>,
|
||||
}
|
||||
|
||||
/// One exact project file inventory entry.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct InventoryFile {
|
||||
/// UTF-8 relative path beneath `project`.
|
||||
pub path: String,
|
||||
/// Exact file length.
|
||||
pub size_bytes: u64,
|
||||
/// SHA-256 of exact file bytes.
|
||||
pub sha256: Digest,
|
||||
}
|
||||
|
||||
/// Outcome of moving corrupt active Analysis data out of visibility.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct QuarantineOutcome {
|
||||
/// Unique retained quarantine directory.
|
||||
pub path: PathBuf,
|
||||
/// Required next decision after successful quarantine.
|
||||
pub decision: RebuildDecision,
|
||||
}
|
||||
|
||||
/// Single-rebuild state machine primitive.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum RebuildDecision {
|
||||
/// One fresh build is permitted.
|
||||
RebuildOnce,
|
||||
/// Recovery was already attempted; surface corruption.
|
||||
FailCorrupt,
|
||||
}
|
||||
|
||||
/// Returns the only permitted recovery action for the invocation state.
|
||||
#[must_use]
|
||||
pub const fn rebuild_decision(rebuild_already_attempted: bool) -> RebuildDecision {
|
||||
if rebuild_already_attempted {
|
||||
RebuildDecision::FailCorrupt
|
||||
} else {
|
||||
RebuildDecision::RebuildOnce
|
||||
}
|
||||
}
|
||||
|
||||
/// Produces the canonical Profile digest after enforcing sorted unique options.
|
||||
pub fn analysis_profile_digest(profile: &AnalysisProfile) -> Result<Digest, StoreError> {
|
||||
if profile.profile_version != 1 {
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::CorruptStore,
|
||||
"Analysis Profile version must be exactly 1",
|
||||
));
|
||||
}
|
||||
validate_options(&profile.loader_options, "loader")?;
|
||||
validate_options(&profile.analyzer_options, "analyzer")?;
|
||||
let canonical = serde_json::to_value(profile)
|
||||
.and_then(|value| serde_json::to_vec(&value))
|
||||
.map_err(|error| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::CorruptStore,
|
||||
format!("Profile serialization failed: {error}"),
|
||||
)
|
||||
})?;
|
||||
let digest = Sha256::digest(&canonical);
|
||||
digest_bytes(&digest)
|
||||
}
|
||||
|
||||
/// Inventories a complete project tree in manifest bytewise path order.
|
||||
pub fn inventory_project(project: &Path) -> Result<Vec<InventoryFile>, StoreError> {
|
||||
reject_tree_root(project)?;
|
||||
inventory_tree(project, project)
|
||||
}
|
||||
|
||||
/// Atomically writes the completed manifest into a private staged Analysis directory.
|
||||
pub fn write_analysis_manifest(
|
||||
staged_analysis: &Path,
|
||||
manifest: &AnalysisManifest,
|
||||
) -> Result<(), StoreError> {
|
||||
super::filesystem::atomic_json_write(
|
||||
staged_analysis,
|
||||
&staged_analysis.join("manifest.json"),
|
||||
manifest,
|
||||
)
|
||||
}
|
||||
|
||||
fn validate_options(
|
||||
options: &[crate::operation::ProfileOption],
|
||||
kind: &str,
|
||||
) -> Result<(), StoreError> {
|
||||
if options
|
||||
.windows(2)
|
||||
.any(|pair| pair[0].name.as_bytes() >= pair[1].name.as_bytes())
|
||||
{
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::CorruptStore,
|
||||
format!("{kind} options must be bytewise sorted with unique names"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates manifest identity and every regular project file without following symlinks.
|
||||
pub fn validate_analysis(
|
||||
analysis_directory: &Path,
|
||||
expected_sample: &Digest,
|
||||
expected_profile: &Digest,
|
||||
) -> Result<AnalysisManifest, StoreError> {
|
||||
reject_tree_root(analysis_directory)?;
|
||||
let manifest_path = analysis_directory.join("manifest.json");
|
||||
require_regular(&manifest_path)?;
|
||||
let bytes = fs::read(&manifest_path).map_err(|error| StoreError::io(&manifest_path, error))?;
|
||||
let manifest: AnalysisManifest = serde_json::from_slice(&bytes).map_err(|error| {
|
||||
StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&manifest_path,
|
||||
format!("invalid Analysis manifest: {error}"),
|
||||
)
|
||||
})?;
|
||||
if manifest.manifest_version != 1
|
||||
|| manifest.state != "complete"
|
||||
|| &manifest.sample.sha256 != expected_sample
|
||||
|| &manifest.analysis_profile.sha256 != expected_profile
|
||||
|| manifest.project.name != "analysis"
|
||||
|| manifest.project.path != "project"
|
||||
{
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&manifest_path,
|
||||
"Analysis manifest identity or fixed fields do not match its location",
|
||||
));
|
||||
}
|
||||
validate_inventory(&analysis_directory.join("project"), &manifest.project.files)?;
|
||||
let mut root_entries = read_sorted(analysis_directory)?;
|
||||
let root_names: Vec<_> = root_entries
|
||||
.drain(..)
|
||||
.map(|entry| entry.file_name())
|
||||
.collect();
|
||||
if root_names != ["manifest.json", "project"].map(std::ffi::OsString::from) {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
analysis_directory,
|
||||
"Analysis directory contains missing or unexpected entries",
|
||||
));
|
||||
}
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
impl AnalysisStore {
|
||||
/// Validates and atomically promotes a staged complete Analysis.
|
||||
pub fn promote_analysis(
|
||||
&self,
|
||||
staged_analysis: &Path,
|
||||
sample: &Digest,
|
||||
profile: &Digest,
|
||||
lock: &AnalysisLock,
|
||||
) -> Result<PathBuf, StoreError> {
|
||||
if !lock.guards(sample, profile) {
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::AnalysisBusy,
|
||||
"lock token does not guard the promoted Analysis identity",
|
||||
));
|
||||
}
|
||||
validate_analysis(staged_analysis, sample, profile)?;
|
||||
let sample_directory = self.path("analyses").join(sample.as_str());
|
||||
create_private_directory(&sample_directory)?;
|
||||
let destination = sample_directory.join(profile.as_str());
|
||||
if fs::symlink_metadata(&destination).is_ok() {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::ImmutableConflict,
|
||||
&destination,
|
||||
"completed Analysis destination already exists",
|
||||
));
|
||||
}
|
||||
fs::rename(staged_analysis, &destination)
|
||||
.map_err(|error| StoreError::io(&destination, error))?;
|
||||
sync_directory(&sample_directory)?;
|
||||
Ok(destination)
|
||||
}
|
||||
|
||||
/// Atomically removes corrupt Analysis data from active visibility.
|
||||
pub fn quarantine_analysis(
|
||||
&self,
|
||||
sample: &Digest,
|
||||
profile: &Digest,
|
||||
lock: &AnalysisLock,
|
||||
rebuild_already_attempted: bool,
|
||||
) -> Result<QuarantineOutcome, StoreError> {
|
||||
if !lock.guards(sample, profile) {
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::AnalysisBusy,
|
||||
"lock token does not guard the quarantined Analysis identity",
|
||||
));
|
||||
}
|
||||
let active = self
|
||||
.path("analyses")
|
||||
.join(sample.as_str())
|
||||
.join(profile.as_str());
|
||||
let parent = self
|
||||
.path("quarantine")
|
||||
.join(sample.as_str())
|
||||
.join(profile.as_str());
|
||||
create_private_directory(&parent)?;
|
||||
let destination = parent.join(random_id()?);
|
||||
fs::rename(&active, &destination).map_err(|error| StoreError::io(&active, error))?;
|
||||
sync_directory(&parent)?;
|
||||
Ok(QuarantineOutcome {
|
||||
path: destination,
|
||||
decision: rebuild_decision(rebuild_already_attempted),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_inventory(project: &Path, recorded: &[InventoryFile]) -> Result<(), StoreError> {
|
||||
reject_tree_root(project)?;
|
||||
if recorded
|
||||
.windows(2)
|
||||
.any(|pair| pair[0].path.as_bytes() >= pair[1].path.as_bytes())
|
||||
{
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
project,
|
||||
"project inventory is not bytewise sorted and unique",
|
||||
));
|
||||
}
|
||||
let actual = inventory_tree(project, project)?;
|
||||
if actual != recorded {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
project,
|
||||
"project inventory size, digest, or membership mismatch",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn inventory_tree(root: &Path, directory: &Path) -> Result<Vec<InventoryFile>, StoreError> {
|
||||
let mut inventory = Vec::new();
|
||||
for entry in read_sorted(directory)? {
|
||||
let path = entry.path();
|
||||
let metadata = fs::symlink_metadata(&path).map_err(|error| StoreError::io(&path, error))?;
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&path,
|
||||
"symlink in Analysis project",
|
||||
));
|
||||
}
|
||||
if metadata.is_dir() {
|
||||
inventory.extend(inventory_tree(root, &path)?);
|
||||
} else if metadata.is_file() {
|
||||
let relative = path.strip_prefix(root).map_err(|_| {
|
||||
StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&path,
|
||||
"project path escaped inventory root",
|
||||
)
|
||||
})?;
|
||||
if relative
|
||||
.components()
|
||||
.any(|component| !matches!(component, Component::Normal(_)))
|
||||
{
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&path,
|
||||
"invalid inventory path",
|
||||
));
|
||||
}
|
||||
let Some(relative) = relative.to_str() else {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&path,
|
||||
"non-UTF-8 project filename",
|
||||
));
|
||||
};
|
||||
inventory.push(InventoryFile {
|
||||
path: relative.to_owned(),
|
||||
size_bytes: metadata.len(),
|
||||
sha256: hash_file(&path)?,
|
||||
});
|
||||
} else {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&path,
|
||||
"special file in Analysis project",
|
||||
));
|
||||
}
|
||||
}
|
||||
inventory.sort_by(|left, right| left.path.as_bytes().cmp(right.path.as_bytes()));
|
||||
Ok(inventory)
|
||||
}
|
||||
|
||||
pub(crate) fn hash_file(path: &Path) -> Result<Digest, StoreError> {
|
||||
let mut file = File::open(path).map_err(|error| StoreError::io(path, error))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buffer = vec![0_u8; 1024 * 1024];
|
||||
loop {
|
||||
let count = file
|
||||
.read(&mut buffer)
|
||||
.map_err(|error| StoreError::io(path, error))?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..count]);
|
||||
}
|
||||
digest_bytes(&hasher.finalize())
|
||||
}
|
||||
|
||||
pub(crate) fn digest_bytes(bytes: &[u8]) -> Result<Digest, StoreError> {
|
||||
Digest::from_str(&hex::encode(bytes)).map_err(|error| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::Io,
|
||||
format!("computed digest was invalid: {error}"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn reject_tree_root(path: &Path) -> Result<(), StoreError> {
|
||||
let metadata = fs::symlink_metadata(path).map_err(|error| StoreError::io(path, error))?;
|
||||
if !metadata.is_dir() || metadata.file_type().is_symlink() {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
path,
|
||||
"expected ordinary directory",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_regular(path: &Path) -> Result<(), StoreError> {
|
||||
let metadata = fs::symlink_metadata(path).map_err(|error| StoreError::io(path, error))?;
|
||||
if !metadata.is_file() || metadata.file_type().is_symlink() {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
path,
|
||||
"expected ordinary file",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_sorted(path: &Path) -> Result<Vec<fs::DirEntry>, StoreError> {
|
||||
let mut entries = fs::read_dir(path)
|
||||
.map_err(|error| StoreError::io(path, error))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|error| StoreError::io(path, error))?;
|
||||
entries.sort_by_key(fs::DirEntry::file_name);
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used)]
|
||||
mod tests {
|
||||
use std::{collections::BTreeMap, fs, str::FromStr as _};
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::{
|
||||
AnalysisManifest, AnalysisProfileManifest, AnalysisProject, CreatedBy, InventoryFile,
|
||||
ManifestSample, RebuildDecision, hash_file, rebuild_decision, validate_analysis,
|
||||
};
|
||||
use crate::{
|
||||
domain::Digest,
|
||||
store::{AnalysisLock, AnalysisStore, StoreEnvironment, resolve_store},
|
||||
};
|
||||
|
||||
fn digest(byte: char) -> Digest {
|
||||
Digest::from_str(&byte.to_string().repeat(64)).expect("digest")
|
||||
}
|
||||
|
||||
fn write_analysis(root: &Path, sample: &Digest, profile: &Digest) {
|
||||
let project = root.join("project");
|
||||
fs::create_dir_all(&project).expect("project");
|
||||
let project_file = project.join("analysis.gpr");
|
||||
fs::write(&project_file, b"project").expect("project file");
|
||||
let manifest = AnalysisManifest {
|
||||
manifest_version: 1,
|
||||
state: "complete".to_owned(),
|
||||
sample: ManifestSample {
|
||||
sha256: sample.clone(),
|
||||
size_bytes: 7,
|
||||
},
|
||||
analysis_profile: AnalysisProfileManifest {
|
||||
sha256: profile.clone(),
|
||||
ghidra_version: "12.1.2".to_owned(),
|
||||
adapter_protocol_version: 1,
|
||||
target: BTreeMap::new(),
|
||||
analyzer_options_sha256: digest('c'),
|
||||
},
|
||||
created_by: CreatedBy {
|
||||
ghidr_version: "0.1.0".to_owned(),
|
||||
invocation_id: "0".repeat(32),
|
||||
completed_at: "2026-01-01T00:00:00Z".to_owned(),
|
||||
},
|
||||
project: AnalysisProject {
|
||||
name: "analysis".to_owned(),
|
||||
path: "project".to_owned(),
|
||||
files: vec![InventoryFile {
|
||||
path: "analysis.gpr".to_owned(),
|
||||
size_bytes: 7,
|
||||
sha256: hash_file(&project_file).expect("hash"),
|
||||
}],
|
||||
},
|
||||
};
|
||||
fs::write(
|
||||
root.join("manifest.json"),
|
||||
serde_json::to_vec(&manifest).expect("json"),
|
||||
)
|
||||
.expect("manifest");
|
||||
}
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
#[test]
|
||||
fn complete_inventory_validates_and_mutation_is_corruption() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let sample = digest('a');
|
||||
let profile = digest('b');
|
||||
write_analysis(temp.path(), &sample, &profile);
|
||||
validate_analysis(temp.path(), &sample, &profile).expect("valid");
|
||||
fs::write(temp.path().join("project/analysis.gpr"), b"changed").expect("change");
|
||||
assert!(validate_analysis(temp.path(), &sample, &profile).is_err());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn inventory_rejects_symlink_attack() {
|
||||
use std::os::unix::fs::symlink;
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let sample = digest('a');
|
||||
let profile = digest('b');
|
||||
write_analysis(temp.path(), &sample, &profile);
|
||||
symlink("analysis.gpr", temp.path().join("project/escape")).expect("symlink");
|
||||
assert!(validate_analysis(temp.path(), &sample, &profile).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_allows_exactly_one_rebuild() {
|
||||
assert_eq!(rebuild_decision(false), RebuildDecision::RebuildOnce);
|
||||
assert_eq!(rebuild_decision(true), RebuildDecision::FailCorrupt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn promotion_is_atomic_and_corruption_moves_to_quarantine() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(temp.path().join("store").into_os_string()),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
let store = AnalysisStore::initialize(resolve_store(None, &environment).expect("resolve"))
|
||||
.expect("store");
|
||||
let sample = digest('a');
|
||||
let profile = digest('b');
|
||||
let staged = store.root().join("staging/build/analysis");
|
||||
fs::create_dir_all(&staged).expect("staging");
|
||||
write_analysis(&staged, &sample, &profile);
|
||||
let lock = AnalysisLock::acquire(&store, &sample, &profile).expect("lock");
|
||||
let active = store
|
||||
.promote_analysis(&staged, &sample, &profile, &lock)
|
||||
.expect("promote");
|
||||
assert!(!staged.exists());
|
||||
validate_analysis(&active, &sample, &profile).expect("active");
|
||||
fs::write(active.join("project/analysis.gpr"), b"corrupt").expect("corrupt");
|
||||
assert!(validate_analysis(&active, &sample, &profile).is_err());
|
||||
let outcome = store
|
||||
.quarantine_analysis(&sample, &profile, &lock, false)
|
||||
.expect("quarantine");
|
||||
assert!(!active.exists());
|
||||
assert!(outcome.path.exists());
|
||||
assert_eq!(outcome.decision, RebuildDecision::RebuildOnce);
|
||||
}
|
||||
}
|
||||
196
src/store/artifact.rs
Normal file
196
src/store/artifact.rs
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
use std::{
|
||||
fs,
|
||||
io::Write as _,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use crate::{
|
||||
domain::{Digest, TaggedPath},
|
||||
operation::ArtifactDescriptor,
|
||||
};
|
||||
|
||||
use super::{
|
||||
AnalysisStore, StoreError, StoreErrorKind,
|
||||
analysis::{digest_bytes, hash_file},
|
||||
filesystem::{
|
||||
create_private_directory, open_new_private_file, random_id, set_file_mode, sync_directory,
|
||||
},
|
||||
};
|
||||
|
||||
/// Published immutable complete-success Artifact.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct Artifact {
|
||||
path: PathBuf,
|
||||
bytes: u64,
|
||||
digest: Digest,
|
||||
}
|
||||
|
||||
impl Artifact {
|
||||
/// Absolute published JSON path.
|
||||
#[must_use]
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Exact bytes including the required trailing LF.
|
||||
#[must_use]
|
||||
pub const fn bytes(&self) -> u64 {
|
||||
self.bytes
|
||||
}
|
||||
|
||||
/// SHA-256 of the exact published bytes.
|
||||
#[must_use]
|
||||
pub const fn digest(&self) -> &Digest {
|
||||
&self.digest
|
||||
}
|
||||
|
||||
/// Builds the fixed public spill descriptor for this Artifact.
|
||||
#[must_use]
|
||||
pub fn descriptor(&self) -> ArtifactDescriptor {
|
||||
ArtifactDescriptor {
|
||||
path: TaggedPath::from_path(&self.path),
|
||||
bytes: self.bytes,
|
||||
sha256: self.digest.clone(),
|
||||
media_type: "application/json".to_owned(),
|
||||
contains: "complete_success_response".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically publishes an exact complete JSON success document by content digest.
|
||||
pub fn publish_artifact(
|
||||
store: &AnalysisStore,
|
||||
sample: &Digest,
|
||||
profile: &Digest,
|
||||
exact_bytes: &[u8],
|
||||
) -> Result<Artifact, StoreError> {
|
||||
validate_artifact_bytes(exact_bytes)?;
|
||||
let digest = digest_bytes(&Sha256::digest(exact_bytes))?;
|
||||
let directory = store
|
||||
.path("artifacts")
|
||||
.join(sample.as_str())
|
||||
.join(profile.as_str());
|
||||
create_private_directory(&directory)?;
|
||||
let destination = directory.join(format!("{}.json", digest.as_str()));
|
||||
if let Ok(metadata) = fs::symlink_metadata(&destination) {
|
||||
if !metadata.is_file()
|
||||
|| metadata.file_type().is_symlink()
|
||||
|| hash_file(&destination)? != digest
|
||||
{
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::ImmutableConflict,
|
||||
&destination,
|
||||
"Artifact identity conflicts with existing storage",
|
||||
));
|
||||
}
|
||||
return artifact(destination, exact_bytes.len(), digest);
|
||||
}
|
||||
|
||||
let temporary = directory.join(format!("artifact-{}.tmp", random_id()?));
|
||||
let result = (|| {
|
||||
let mut file = open_new_private_file(&temporary)?;
|
||||
file.write_all(exact_bytes)
|
||||
.map_err(|error| StoreError::io(&temporary, error))?;
|
||||
file.sync_all()
|
||||
.map_err(|error| StoreError::io(&temporary, error))?;
|
||||
set_file_mode(&temporary, 0o400)?;
|
||||
match fs::hard_link(&temporary, &destination) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
|
||||
if hash_file(&destination)? != digest {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::ImmutableConflict,
|
||||
&destination,
|
||||
"concurrent Artifact conflict",
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(StoreError::io(&destination, error)),
|
||||
}
|
||||
fs::remove_file(&temporary).map_err(|error| StoreError::io(&temporary, error))?;
|
||||
sync_directory(&directory)
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(&temporary);
|
||||
}
|
||||
result?;
|
||||
artifact(destination, exact_bytes.len(), digest)
|
||||
}
|
||||
|
||||
fn validate_artifact_bytes(bytes: &[u8]) -> Result<(), StoreError> {
|
||||
if bytes.is_empty() || !bytes.ends_with(b"\n") || bytes.ends_with(b"\n\n") {
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::Io,
|
||||
"Artifact must have exactly one trailing LF",
|
||||
));
|
||||
}
|
||||
let json = &bytes[..bytes.len() - 1];
|
||||
if json.starts_with(&[0xef, 0xbb, 0xbf])
|
||||
|| std::str::from_utf8(json).is_err()
|
||||
|| serde_json::from_slice::<serde_json::Value>(json).is_err()
|
||||
{
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::Io,
|
||||
"Artifact must be BOM-free valid UTF-8 JSON",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn artifact(path: PathBuf, length: usize, digest: Digest) -> Result<Artifact, StoreError> {
|
||||
Ok(Artifact {
|
||||
path,
|
||||
bytes: u64::try_from(length)
|
||||
.map_err(|_| StoreError::new(StoreErrorKind::Io, "Artifact length overflow"))?,
|
||||
digest,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used)]
|
||||
mod tests {
|
||||
use super::publish_artifact;
|
||||
use crate::{
|
||||
domain::Digest,
|
||||
store::{AnalysisStore, StoreEnvironment, resolve_store},
|
||||
};
|
||||
use std::{ffi::OsString, fs, str::FromStr as _};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn identical_exact_bytes_publish_one_immutable_identity() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(OsString::from(temp.path().join("store"))),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
let store = AnalysisStore::initialize(resolve_store(None, &environment).expect("resolve"))
|
||||
.expect("store");
|
||||
let sample = Digest::from_str(&"a".repeat(64)).expect("digest");
|
||||
let profile = Digest::from_str(&"b".repeat(64)).expect("digest");
|
||||
let first = publish_artifact(&store, &sample, &profile, b"{\"kind\":\"inspection\"}\n")
|
||||
.expect("publish");
|
||||
let second = publish_artifact(&store, &sample, &profile, b"{\"kind\":\"inspection\"}\n")
|
||||
.expect("reuse");
|
||||
assert_eq!(first, second);
|
||||
assert_eq!(
|
||||
fs::read(first.path()).expect("read"),
|
||||
b"{\"kind\":\"inspection\"}\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_partial_or_non_json_artifacts() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(OsString::from(temp.path().join("store"))),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
let store = AnalysisStore::initialize(resolve_store(None, &environment).expect("resolve"))
|
||||
.expect("store");
|
||||
let digest = Digest::from_str(&"a".repeat(64)).expect("digest");
|
||||
assert!(publish_artifact(&store, &digest, &digest, b"{}").is_err());
|
||||
}
|
||||
}
|
||||
738
src/store/cleanup.rs
Normal file
738
src/store/cleanup.rs
Normal file
|
|
@ -0,0 +1,738 @@
|
|||
use std::{
|
||||
collections::BTreeSet,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
str::FromStr as _,
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
domain::Digest,
|
||||
operation::{CleanupSnapshot, StorageUsage},
|
||||
};
|
||||
|
||||
use super::{
|
||||
AnalysisLock, AnalysisStore, StoreError, StoreErrorKind,
|
||||
filesystem::{
|
||||
atomic_json_write, create_private_directory, random_id, scan_usage, sync_directory,
|
||||
},
|
||||
};
|
||||
|
||||
/// Store-wide or one-Sample cleanup selection.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum CleanupScope {
|
||||
/// All data tied to this Sample digest.
|
||||
Sample(Digest),
|
||||
/// Every removable immutable object and Diagnostic Log.
|
||||
All,
|
||||
}
|
||||
|
||||
/// Deterministic non-mutating cleanup preflight, including resumable staged data.
|
||||
#[derive(Debug)]
|
||||
pub struct CleanupPlan {
|
||||
scope: CleanupScope,
|
||||
entries: Vec<PlanEntry>,
|
||||
snapshot: CleanupSnapshot,
|
||||
}
|
||||
|
||||
impl CleanupPlan {
|
||||
/// Exact counts, usage, and sorted Profile identities selected by preflight.
|
||||
#[must_use]
|
||||
pub const fn snapshot(&self) -> &CleanupSnapshot {
|
||||
&self.snapshot
|
||||
}
|
||||
|
||||
/// Executes locking, atomic staging, deletion, and transaction recovery.
|
||||
pub fn execute(self, store: &AnalysisStore) -> Result<CleanupTransaction, StoreError> {
|
||||
execute_plan(store, self)
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of an executed cleanup transaction.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct CleanupTransaction {
|
||||
/// Exact preflight snapshot.
|
||||
pub matched: CleanupSnapshot,
|
||||
/// Exact successfully removed subset.
|
||||
pub removed: CleanupSnapshot,
|
||||
/// Retained transaction path only when deletion is incomplete.
|
||||
pub transaction_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum Category {
|
||||
Analysis,
|
||||
Quarantine,
|
||||
Artifact,
|
||||
Diagnostic,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum EntryState {
|
||||
Active,
|
||||
Staged,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct PlanEntry {
|
||||
path: PathBuf,
|
||||
original: PathBuf,
|
||||
state: EntryState,
|
||||
category: Category,
|
||||
sample: Option<Digest>,
|
||||
profile: Option<Digest>,
|
||||
usage: StorageUsage,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct TransactionManifest {
|
||||
transaction_version: u32,
|
||||
scope: TransactionScope,
|
||||
entries: Vec<TransactionEntry>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(tag = "kind", content = "sha256", rename_all = "snake_case")]
|
||||
enum TransactionScope {
|
||||
Sample(Digest),
|
||||
All,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct TransactionEntry {
|
||||
staged_name: String,
|
||||
original_path: String,
|
||||
category: Category,
|
||||
sample: Option<Digest>,
|
||||
profile: Option<Digest>,
|
||||
usage: StorageUsage,
|
||||
}
|
||||
|
||||
/// Traverses active and previously staged objects without changing store data.
|
||||
pub fn plan_cleanup(store: &AnalysisStore, scope: CleanupScope) -> Result<CleanupPlan, StoreError> {
|
||||
let mut entries = active_entries(store, &scope)?;
|
||||
entries.extend(staged_entries(store, &scope)?);
|
||||
entries.sort_by(|left, right| path_bytes(&left.path).cmp(path_bytes(&right.path)));
|
||||
let snapshot = summarize(&entries)?;
|
||||
Ok(CleanupPlan {
|
||||
scope,
|
||||
entries,
|
||||
snapshot,
|
||||
})
|
||||
}
|
||||
|
||||
fn active_entries(
|
||||
store: &AnalysisStore,
|
||||
scope: &CleanupScope,
|
||||
) -> Result<Vec<PlanEntry>, StoreError> {
|
||||
let mut entries = Vec::new();
|
||||
collect_profile_directories(
|
||||
store,
|
||||
scope,
|
||||
"analyses",
|
||||
Category::Analysis,
|
||||
false,
|
||||
&mut entries,
|
||||
)?;
|
||||
collect_profile_directories(
|
||||
store,
|
||||
scope,
|
||||
"quarantine",
|
||||
Category::Quarantine,
|
||||
true,
|
||||
&mut entries,
|
||||
)?;
|
||||
collect_artifacts(store, scope, &mut entries)?;
|
||||
collect_diagnostics(store, scope, &mut entries)?;
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
fn collect_profile_directories(
|
||||
store: &AnalysisStore,
|
||||
scope: &CleanupScope,
|
||||
top: &str,
|
||||
category: Category,
|
||||
has_invocations: bool,
|
||||
entries: &mut Vec<PlanEntry>,
|
||||
) -> Result<(), StoreError> {
|
||||
for sample_entry in selected_sample_directories(&store.path(top), scope)? {
|
||||
let sample = parse_digest_name(&sample_entry)?;
|
||||
for profile_entry in directories(&sample_entry)? {
|
||||
let profile = parse_digest_name(&profile_entry)?;
|
||||
if has_invocations {
|
||||
for invocation in directories(&profile_entry)? {
|
||||
push_active(
|
||||
entries,
|
||||
invocation,
|
||||
category,
|
||||
Some(sample.clone()),
|
||||
Some(profile.clone()),
|
||||
)?;
|
||||
}
|
||||
} else {
|
||||
push_active(
|
||||
entries,
|
||||
profile_entry,
|
||||
category,
|
||||
Some(sample.clone()),
|
||||
Some(profile),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_artifacts(
|
||||
store: &AnalysisStore,
|
||||
scope: &CleanupScope,
|
||||
entries: &mut Vec<PlanEntry>,
|
||||
) -> Result<(), StoreError> {
|
||||
for sample_entry in selected_sample_directories(&store.path("artifacts"), scope)? {
|
||||
let sample = parse_digest_name(&sample_entry)?;
|
||||
for profile_entry in directories(&sample_entry)? {
|
||||
let profile = parse_digest_name(&profile_entry)?;
|
||||
for artifact in regular_files(&profile_entry)? {
|
||||
push_active(
|
||||
entries,
|
||||
artifact,
|
||||
Category::Artifact,
|
||||
Some(sample.clone()),
|
||||
Some(profile.clone()),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_diagnostics(
|
||||
store: &AnalysisStore,
|
||||
scope: &CleanupScope,
|
||||
entries: &mut Vec<PlanEntry>,
|
||||
) -> Result<(), StoreError> {
|
||||
let root = store.path("diagnostics");
|
||||
for directory in directories(&root)? {
|
||||
let name = file_name_utf8(&directory)?;
|
||||
let sample = if name == "global" {
|
||||
if !matches!(scope, CleanupScope::All) {
|
||||
continue;
|
||||
}
|
||||
None
|
||||
} else {
|
||||
let digest = parse_digest_name(&directory)?;
|
||||
if !scope_matches(scope, &digest) {
|
||||
continue;
|
||||
}
|
||||
Some(digest)
|
||||
};
|
||||
for bundle in directories(&directory)? {
|
||||
push_active(entries, bundle, Category::Diagnostic, sample.clone(), None)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn push_active(
|
||||
entries: &mut Vec<PlanEntry>,
|
||||
path: PathBuf,
|
||||
category: Category,
|
||||
sample: Option<Digest>,
|
||||
profile: Option<Digest>,
|
||||
) -> Result<(), StoreError> {
|
||||
let usage = scan_usage(&path)?;
|
||||
entries.push(PlanEntry {
|
||||
original: path.clone(),
|
||||
path,
|
||||
state: EntryState::Active,
|
||||
category,
|
||||
sample,
|
||||
profile,
|
||||
usage,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn staged_entries(
|
||||
store: &AnalysisStore,
|
||||
scope: &CleanupScope,
|
||||
) -> Result<Vec<PlanEntry>, StoreError> {
|
||||
let mut result = Vec::new();
|
||||
for transaction in directories(&store.path("cleanup"))? {
|
||||
let manifest_path = transaction.join("manifest.json");
|
||||
let bytes =
|
||||
fs::read(&manifest_path).map_err(|error| StoreError::io(&manifest_path, error))?;
|
||||
let manifest: TransactionManifest = serde_json::from_slice(&bytes).map_err(|error| {
|
||||
StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&manifest_path,
|
||||
format!("invalid cleanup manifest: {error}"),
|
||||
)
|
||||
})?;
|
||||
if manifest.transaction_version != 1 || !transaction_scope_matches(&manifest.scope, scope) {
|
||||
continue;
|
||||
}
|
||||
for entry in manifest.entries {
|
||||
let path = transaction.join("payload").join(&entry.staged_name);
|
||||
match fs::symlink_metadata(&path) {
|
||||
Ok(_) => result.push(PlanEntry {
|
||||
path,
|
||||
original: PathBuf::from(entry.original_path),
|
||||
state: EntryState::Staged,
|
||||
category: entry.category,
|
||||
sample: entry.sample,
|
||||
profile: entry.profile,
|
||||
usage: entry.usage,
|
||||
}),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(StoreError::io(&path, error)),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn execute_plan(
|
||||
store: &AnalysisStore,
|
||||
plan: CleanupPlan,
|
||||
) -> Result<CleanupTransaction, StoreError> {
|
||||
let mut lock_keys: BTreeSet<(Digest, Digest)> = BTreeSet::new();
|
||||
for entry in &plan.entries {
|
||||
if let (Some(sample), Some(profile)) = (&entry.sample, &entry.profile) {
|
||||
lock_keys.insert((sample.clone(), profile.clone()));
|
||||
}
|
||||
}
|
||||
let mut locks = Vec::new();
|
||||
for (sample, profile) in lock_keys {
|
||||
locks.push(AnalysisLock::acquire(store, &sample, &profile)?);
|
||||
}
|
||||
|
||||
let staged_existing: Vec<_> = plan
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|entry| matches!(entry.state, EntryState::Staged))
|
||||
.cloned()
|
||||
.collect();
|
||||
let active: Vec<_> = plan
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|entry| matches!(entry.state, EntryState::Active))
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut removed_entries = Vec::new();
|
||||
for entry in staged_existing {
|
||||
remove_entry(&entry.path).map_err(|error| cleanup_incomplete(&entry.path, error))?;
|
||||
removed_entries.push(entry);
|
||||
}
|
||||
remove_empty_transactions(store)?;
|
||||
|
||||
let transaction_path = if active.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(stage_active(store, &plan.scope, &active)?)
|
||||
};
|
||||
if let Some(transaction) = &transaction_path {
|
||||
let staged = staged_entries_in_transaction(transaction)?;
|
||||
for entry in staged {
|
||||
if let Err(error) = remove_entry(&entry.path) {
|
||||
let removed = summarize(&removed_entries)?;
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CleanupIncomplete,
|
||||
transaction,
|
||||
format!(
|
||||
"cleanup deletion incomplete after removing {} objects: {error}",
|
||||
count_objects(&removed)
|
||||
),
|
||||
));
|
||||
}
|
||||
removed_entries.push(entry);
|
||||
}
|
||||
fs::remove_dir_all(transaction).map_err(|error| cleanup_incomplete(transaction, error))?;
|
||||
sync_directory(&store.path("cleanup"))?;
|
||||
}
|
||||
drop(locks);
|
||||
Ok(CleanupTransaction {
|
||||
matched: plan.snapshot,
|
||||
removed: summarize(&removed_entries)?,
|
||||
transaction_path: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn stage_active(
|
||||
store: &AnalysisStore,
|
||||
scope: &CleanupScope,
|
||||
entries: &[PlanEntry],
|
||||
) -> Result<PathBuf, StoreError> {
|
||||
let transaction = store.path("cleanup").join(random_id()?);
|
||||
let payload = transaction.join("payload");
|
||||
create_private_directory(&payload)?;
|
||||
let manifest_entries: Vec<_> = entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, entry)| TransactionEntry {
|
||||
staged_name: format!("{index:016x}"),
|
||||
original_path: entry.original.to_string_lossy().into_owned(),
|
||||
category: entry.category,
|
||||
sample: entry.sample.clone(),
|
||||
profile: entry.profile.clone(),
|
||||
usage: entry.usage,
|
||||
})
|
||||
.collect();
|
||||
let manifest = TransactionManifest {
|
||||
transaction_version: 1,
|
||||
scope: match scope {
|
||||
CleanupScope::Sample(digest) => TransactionScope::Sample(digest.clone()),
|
||||
CleanupScope::All => TransactionScope::All,
|
||||
},
|
||||
entries: manifest_entries,
|
||||
};
|
||||
atomic_json_write(&transaction, &transaction.join("manifest.json"), &manifest)?;
|
||||
let mut completed = Vec::new();
|
||||
for (index, entry) in entries.iter().enumerate() {
|
||||
let staged = payload.join(format!("{index:016x}"));
|
||||
if let Err(error) = fs::rename(&entry.path, &staged) {
|
||||
for (original, staged) in completed.into_iter().rev() {
|
||||
let _ = fs::rename(staged, original);
|
||||
}
|
||||
let _ = fs::remove_dir_all(&transaction);
|
||||
return Err(StoreError::io(&entry.path, error));
|
||||
}
|
||||
completed.push((entry.original.clone(), staged));
|
||||
}
|
||||
sync_directory(&payload)?;
|
||||
Ok(transaction)
|
||||
}
|
||||
|
||||
fn staged_entries_in_transaction(transaction: &Path) -> Result<Vec<PlanEntry>, StoreError> {
|
||||
let manifest_path = transaction.join("manifest.json");
|
||||
let manifest: TransactionManifest = serde_json::from_slice(
|
||||
&fs::read(&manifest_path).map_err(|error| StoreError::io(&manifest_path, error))?,
|
||||
)
|
||||
.map_err(|error| {
|
||||
StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&manifest_path,
|
||||
format!("invalid cleanup manifest: {error}"),
|
||||
)
|
||||
})?;
|
||||
Ok(manifest
|
||||
.entries
|
||||
.into_iter()
|
||||
.map(|entry| PlanEntry {
|
||||
path: transaction.join("payload").join(entry.staged_name),
|
||||
original: PathBuf::from(entry.original_path),
|
||||
state: EntryState::Staged,
|
||||
category: entry.category,
|
||||
sample: entry.sample,
|
||||
profile: entry.profile,
|
||||
usage: entry.usage,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn remove_entry(path: &Path) -> std::io::Result<()> {
|
||||
let metadata = fs::symlink_metadata(path)?;
|
||||
if metadata.is_dir() {
|
||||
fs::remove_dir_all(path)
|
||||
} else {
|
||||
fs::remove_file(path)
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_empty_transactions(store: &AnalysisStore) -> Result<(), StoreError> {
|
||||
for transaction in directories(&store.path("cleanup"))? {
|
||||
let payload = transaction.join("payload");
|
||||
if directories_and_files(&payload)?.is_empty() {
|
||||
fs::remove_dir_all(&transaction)
|
||||
.map_err(|error| StoreError::io(&transaction, error))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn summarize(entries: &[PlanEntry]) -> Result<CleanupSnapshot, StoreError> {
|
||||
let mut result = CleanupSnapshot {
|
||||
analyses: 0,
|
||||
quarantined_analyses: 0,
|
||||
artifacts: 0,
|
||||
diagnostic_logs: 0,
|
||||
usage: StorageUsage {
|
||||
logical_bytes: 0,
|
||||
allocated_bytes: 0,
|
||||
},
|
||||
analysis_profile_sha256: Vec::new(),
|
||||
};
|
||||
let mut profiles = BTreeSet::new();
|
||||
for entry in entries {
|
||||
match entry.category {
|
||||
Category::Analysis => result.analyses += 1,
|
||||
Category::Quarantine => result.quarantined_analyses += 1,
|
||||
Category::Artifact => result.artifacts += 1,
|
||||
Category::Diagnostic => result.diagnostic_logs += 1,
|
||||
}
|
||||
result.usage.logical_bytes = result
|
||||
.usage
|
||||
.logical_bytes
|
||||
.checked_add(entry.usage.logical_bytes)
|
||||
.ok_or_else(|| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::CorruptStore,
|
||||
"cleanup logical usage overflow",
|
||||
)
|
||||
})?;
|
||||
result.usage.allocated_bytes = result
|
||||
.usage
|
||||
.allocated_bytes
|
||||
.checked_add(entry.usage.allocated_bytes)
|
||||
.ok_or_else(|| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::CorruptStore,
|
||||
"cleanup allocated usage overflow",
|
||||
)
|
||||
})?;
|
||||
if let Some(profile) = &entry.profile {
|
||||
profiles.insert(profile.clone());
|
||||
}
|
||||
}
|
||||
result.analysis_profile_sha256 = profiles.into_iter().collect();
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn selected_sample_directories(
|
||||
root: &Path,
|
||||
scope: &CleanupScope,
|
||||
) -> Result<Vec<PathBuf>, StoreError> {
|
||||
let directories = directories(root)?;
|
||||
directories
|
||||
.into_iter()
|
||||
.filter_map(|path| match parse_digest_name(&path) {
|
||||
Ok(digest) if scope_matches(scope, &digest) => Some(Ok(path)),
|
||||
Ok(_) => None,
|
||||
Err(error) => Some(Err(error)),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn directories(path: &Path) -> Result<Vec<PathBuf>, StoreError> {
|
||||
filter_entries(path, true)
|
||||
}
|
||||
|
||||
fn regular_files(path: &Path) -> Result<Vec<PathBuf>, StoreError> {
|
||||
filter_entries(path, false)
|
||||
}
|
||||
|
||||
fn directories_and_files(path: &Path) -> Result<Vec<PathBuf>, StoreError> {
|
||||
let mut result = Vec::new();
|
||||
for entry in read_entries(path)? {
|
||||
let metadata = fs::symlink_metadata(entry.path())
|
||||
.map_err(|error| StoreError::io(entry.path(), error))?;
|
||||
if metadata.file_type().is_symlink() || !(metadata.is_dir() || metadata.is_file()) {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
entry.path(),
|
||||
"unsafe cleanup entry",
|
||||
));
|
||||
}
|
||||
result.push(entry.path());
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn filter_entries(path: &Path, want_directory: bool) -> Result<Vec<PathBuf>, StoreError> {
|
||||
let mut result = Vec::new();
|
||||
for entry in read_entries(path)? {
|
||||
let metadata = fs::symlink_metadata(entry.path())
|
||||
.map_err(|error| StoreError::io(entry.path(), error))?;
|
||||
if metadata.file_type().is_symlink()
|
||||
|| metadata.is_dir() != want_directory
|
||||
|| metadata.is_file() == want_directory
|
||||
{
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
entry.path(),
|
||||
"unexpected or unsafe store entry",
|
||||
));
|
||||
}
|
||||
result.push(entry.path());
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn read_entries(path: &Path) -> Result<Vec<fs::DirEntry>, StoreError> {
|
||||
let mut entries = fs::read_dir(path)
|
||||
.map_err(|error| StoreError::io(path, error))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|error| StoreError::io(path, error))?;
|
||||
entries.sort_by_key(fs::DirEntry::file_name);
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
fn parse_digest_name(path: &Path) -> Result<Digest, StoreError> {
|
||||
Digest::from_str(file_name_utf8(path)?).map_err(|_| {
|
||||
StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
path,
|
||||
"non-digest store component",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn file_name_utf8(path: &Path) -> Result<&str, StoreError> {
|
||||
path.file_name()
|
||||
.and_then(std::ffi::OsStr::to_str)
|
||||
.ok_or_else(|| {
|
||||
StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
path,
|
||||
"non-UTF-8 generated component",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn scope_matches(scope: &CleanupScope, digest: &Digest) -> bool {
|
||||
matches!(scope, CleanupScope::All)
|
||||
|| matches!(scope, CleanupScope::Sample(selected) if selected == digest)
|
||||
}
|
||||
|
||||
fn transaction_scope_matches(transaction: &TransactionScope, scope: &CleanupScope) -> bool {
|
||||
matches!(
|
||||
(transaction, scope),
|
||||
(TransactionScope::All, CleanupScope::All)
|
||||
) || matches!((transaction, scope), (TransactionScope::Sample(left), CleanupScope::Sample(right)) if left == right)
|
||||
|| matches!(scope, CleanupScope::All)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn path_bytes(path: &Path) -> &[u8] {
|
||||
use std::os::unix::ffi::OsStrExt as _;
|
||||
path.as_os_str().as_bytes()
|
||||
}
|
||||
|
||||
fn cleanup_incomplete(path: &Path, error: std::io::Error) -> StoreError {
|
||||
StoreError::at(
|
||||
StoreErrorKind::CleanupIncomplete,
|
||||
path,
|
||||
format!("cleanup deletion incomplete: {error}"),
|
||||
)
|
||||
}
|
||||
|
||||
fn count_objects(snapshot: &CleanupSnapshot) -> u64 {
|
||||
snapshot.analyses
|
||||
+ snapshot.quarantined_analyses
|
||||
+ snapshot.artifacts
|
||||
+ snapshot.diagnostic_logs
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used)]
|
||||
mod tests {
|
||||
use super::{CleanupScope, plan_cleanup};
|
||||
use crate::{
|
||||
domain::Digest,
|
||||
store::{AnalysisLock, AnalysisStore, StoreEnvironment, resolve_store},
|
||||
};
|
||||
use std::{ffi::OsString, fs, str::FromStr as _};
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn setup() -> (TempDir, AnalysisStore, Digest, Digest) {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(OsString::from(temp.path().join("store"))),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
let store = AnalysisStore::initialize(resolve_store(None, &environment).expect("resolve"))
|
||||
.expect("store");
|
||||
let sample = Digest::from_str(&"a".repeat(64)).expect("digest");
|
||||
let profile = Digest::from_str(&"b".repeat(64)).expect("digest");
|
||||
(temp, store, sample, profile)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dry_run_plan_is_exact_and_non_mutating() {
|
||||
let (_temp, store, sample, profile) = setup();
|
||||
let path = store
|
||||
.root()
|
||||
.join("artifacts")
|
||||
.join(sample.as_str())
|
||||
.join(profile.as_str());
|
||||
fs::create_dir_all(&path).expect("dirs");
|
||||
fs::write(path.join(format!("{}.json", "c".repeat(64))), b"{}\n").expect("artifact");
|
||||
let plan = plan_cleanup(&store, CleanupScope::Sample(sample)).expect("plan");
|
||||
assert_eq!(plan.snapshot().artifacts, 1);
|
||||
assert!(path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_stages_then_removes_selected_objects_only() {
|
||||
let (_temp, store, sample, profile) = setup();
|
||||
let selected = store
|
||||
.root()
|
||||
.join("artifacts")
|
||||
.join(sample.as_str())
|
||||
.join(profile.as_str());
|
||||
fs::create_dir_all(&selected).expect("dirs");
|
||||
fs::write(selected.join(format!("{}.json", "c".repeat(64))), b"{}\n").expect("artifact");
|
||||
let other = Digest::from_str(&"d".repeat(64)).expect("digest");
|
||||
let other_path = store
|
||||
.root()
|
||||
.join("artifacts")
|
||||
.join(other.as_str())
|
||||
.join(profile.as_str());
|
||||
fs::create_dir_all(&other_path).expect("dirs");
|
||||
fs::write(other_path.join(format!("{}.json", "e".repeat(64))), b"{}\n").expect("artifact");
|
||||
let result = plan_cleanup(&store, CleanupScope::Sample(sample))
|
||||
.expect("plan")
|
||||
.execute(&store)
|
||||
.expect("execute");
|
||||
assert_eq!(result.removed.artifacts, 1);
|
||||
assert!(!selected.join(format!("{}.json", "c".repeat(64))).exists());
|
||||
assert!(other_path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn staged_transaction_is_discovered_for_resumable_recovery() {
|
||||
let (_temp, store, sample, profile) = setup();
|
||||
let transaction = store.root().join("cleanup").join("0".repeat(32));
|
||||
fs::create_dir_all(transaction.join("payload")).expect("payload");
|
||||
fs::write(transaction.join("payload/0000000000000000"), b"{}\n").expect("staged");
|
||||
let manifest = serde_json::json!({
|
||||
"transaction_version": 1,
|
||||
"scope": {"kind":"sample", "sha256": sample},
|
||||
"entries": [{"staged_name":"0000000000000000", "original_path":"/gone", "category":"artifact", "sample": sample, "profile": profile, "usage":{"logical_bytes":3,"allocated_bytes":4096}}]
|
||||
});
|
||||
fs::write(
|
||||
transaction.join("manifest.json"),
|
||||
serde_json::to_vec(&manifest).expect("json"),
|
||||
)
|
||||
.expect("manifest");
|
||||
let plan = plan_cleanup(
|
||||
&store,
|
||||
CleanupScope::Sample(Digest::from_str(&"a".repeat(64)).expect("digest")),
|
||||
)
|
||||
.expect("plan");
|
||||
assert_eq!(plan.snapshot().artifacts, 1);
|
||||
plan.execute(&store).expect("resume");
|
||||
assert!(!transaction.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn busy_analysis_aborts_before_store_mutation() {
|
||||
let (_temp, store, sample, profile) = setup();
|
||||
let analysis = store
|
||||
.root()
|
||||
.join("analyses")
|
||||
.join(sample.as_str())
|
||||
.join(profile.as_str());
|
||||
fs::create_dir_all(&analysis).expect("analysis");
|
||||
fs::write(analysis.join("data"), b"active").expect("data");
|
||||
let held = AnalysisLock::acquire(&store, &sample, &profile).expect("lock");
|
||||
let plan = plan_cleanup(&store, CleanupScope::Sample(sample)).expect("plan");
|
||||
assert!(plan.execute(&store).is_err());
|
||||
assert!(analysis.exists());
|
||||
drop(held);
|
||||
}
|
||||
}
|
||||
215
src/store/cleanup_command.rs
Normal file
215
src/store/cleanup_command.rs
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
AppError, ErrorCode,
|
||||
cli::CleanArgs,
|
||||
domain::TaggedPath,
|
||||
operation::{CleanupData, CleanupMode, CleanupTarget},
|
||||
};
|
||||
|
||||
use super::{
|
||||
AnalysisStore, CleanupScope, SnapshotOptions, StoreError, StoreErrorKind, plan_cleanup,
|
||||
snapshot_sample,
|
||||
};
|
||||
|
||||
/// Executes the store-facing portion of the public `clean` command synchronously.
|
||||
pub fn execute_cleanup(
|
||||
store: &AnalysisStore,
|
||||
arguments: &CleanArgs,
|
||||
max_sample_bytes: u64,
|
||||
) -> Result<CleanupData, AppError> {
|
||||
let (scope, target) = if let Some(sample) = &arguments.sample {
|
||||
let snapshot = snapshot_sample(
|
||||
store,
|
||||
sample,
|
||||
SnapshotOptions {
|
||||
max_sample_bytes,
|
||||
reserve_bytes: 0,
|
||||
},
|
||||
)
|
||||
.map_err(app_error)?;
|
||||
(
|
||||
CleanupScope::Sample(snapshot.digest().clone()),
|
||||
CleanupTarget::Sample {
|
||||
source_path: TaggedPath::from_path(sample),
|
||||
sha256: snapshot.digest().clone(),
|
||||
},
|
||||
)
|
||||
} else if let Some(digest) = &arguments.digest {
|
||||
(
|
||||
CleanupScope::Sample(digest.clone()),
|
||||
CleanupTarget::Digest {
|
||||
sha256: digest.clone(),
|
||||
},
|
||||
)
|
||||
} else if arguments.all {
|
||||
(CleanupScope::All, CleanupTarget::All)
|
||||
} else {
|
||||
return Err(AppError::invalid_arguments(
|
||||
"exactly one cleanup target is required",
|
||||
));
|
||||
};
|
||||
|
||||
let plan = plan_cleanup(store, scope.clone()).map_err(app_error)?;
|
||||
if arguments.all && !arguments.dry_run && !arguments.yes {
|
||||
return Err(AppError::new(
|
||||
ErrorCode::ConfirmationRequired,
|
||||
"store-wide cleanup requires explicit confirmation",
|
||||
false,
|
||||
)
|
||||
.with_detail("analyses", Value::from(plan.snapshot().analyses))
|
||||
.with_detail("artifacts", Value::from(plan.snapshot().artifacts))
|
||||
.with_detail(
|
||||
"logical_bytes",
|
||||
Value::from(plan.snapshot().usage.logical_bytes),
|
||||
)
|
||||
.with_detail(
|
||||
"allocated_bytes",
|
||||
Value::from(plan.snapshot().usage.allocated_bytes),
|
||||
));
|
||||
}
|
||||
if arguments.dry_run {
|
||||
return Ok(CleanupData {
|
||||
mode: CleanupMode::DryRun,
|
||||
target,
|
||||
matched: plan.snapshot().clone(),
|
||||
removed: None,
|
||||
});
|
||||
}
|
||||
let matched = plan.snapshot().clone();
|
||||
let transaction = match plan.execute(store) {
|
||||
Ok(transaction) => transaction,
|
||||
Err(error) if error.kind() == StoreErrorKind::CleanupIncomplete => {
|
||||
let remaining = plan_cleanup(store, scope)
|
||||
.map(|plan| plan.snapshot().clone())
|
||||
.unwrap_or_else(|_| matched.clone());
|
||||
let removed = subtract_snapshot(&matched, &remaining);
|
||||
let mut app = app_error(error);
|
||||
app = app.with_detail(
|
||||
"removed",
|
||||
serde_json::to_value(removed).unwrap_or(Value::Null),
|
||||
);
|
||||
app = app.with_detail(
|
||||
"remaining",
|
||||
serde_json::to_value(remaining).unwrap_or(Value::Null),
|
||||
);
|
||||
return Err(app);
|
||||
}
|
||||
Err(error) => return Err(app_error(error)),
|
||||
};
|
||||
Ok(CleanupData {
|
||||
mode: CleanupMode::Executed,
|
||||
target,
|
||||
matched: transaction.matched,
|
||||
removed: Some(transaction.removed),
|
||||
})
|
||||
}
|
||||
|
||||
fn subtract_snapshot(
|
||||
matched: &crate::operation::CleanupSnapshot,
|
||||
remaining: &crate::operation::CleanupSnapshot,
|
||||
) -> crate::operation::CleanupSnapshot {
|
||||
use crate::operation::{CleanupSnapshot, StorageUsage};
|
||||
|
||||
CleanupSnapshot {
|
||||
analyses: matched.analyses.saturating_sub(remaining.analyses),
|
||||
quarantined_analyses: matched
|
||||
.quarantined_analyses
|
||||
.saturating_sub(remaining.quarantined_analyses),
|
||||
artifacts: matched.artifacts.saturating_sub(remaining.artifacts),
|
||||
diagnostic_logs: matched
|
||||
.diagnostic_logs
|
||||
.saturating_sub(remaining.diagnostic_logs),
|
||||
usage: StorageUsage {
|
||||
logical_bytes: matched
|
||||
.usage
|
||||
.logical_bytes
|
||||
.saturating_sub(remaining.usage.logical_bytes),
|
||||
allocated_bytes: matched
|
||||
.usage
|
||||
.allocated_bytes
|
||||
.saturating_sub(remaining.usage.allocated_bytes),
|
||||
},
|
||||
analysis_profile_sha256: matched
|
||||
.analysis_profile_sha256
|
||||
.iter()
|
||||
.filter(|profile| !remaining.analysis_profile_sha256.contains(profile))
|
||||
.cloned()
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn app_error(error: StoreError) -> AppError {
|
||||
let (code, retryable) = match error.kind() {
|
||||
StoreErrorKind::InvalidStorePath | StoreErrorKind::UnsafeStorePath => {
|
||||
(ErrorCode::InvalidStorePath, false)
|
||||
}
|
||||
StoreErrorKind::CorruptStore | StoreErrorKind::ImmutableConflict => {
|
||||
(ErrorCode::CorruptAnalysis, false)
|
||||
}
|
||||
StoreErrorKind::SampleNotFound => (ErrorCode::SampleNotFound, false),
|
||||
StoreErrorKind::SampleUnreadable => (ErrorCode::SampleUnreadable, false),
|
||||
StoreErrorKind::InvalidSampleType => (ErrorCode::InvalidSampleType, false),
|
||||
StoreErrorKind::SampleTooLarge => (ErrorCode::SampleTooLarge, false),
|
||||
StoreErrorKind::SampleChanged => (ErrorCode::SampleChanged, true),
|
||||
StoreErrorKind::InsufficientStoreSpace => (ErrorCode::InsufficientStoreSpace, true),
|
||||
StoreErrorKind::AnalysisBusy => (ErrorCode::AnalysisBusy, true),
|
||||
StoreErrorKind::CleanupIncomplete => (ErrorCode::CleanupIncomplete, true),
|
||||
StoreErrorKind::Io => (ErrorCode::Internal, false),
|
||||
};
|
||||
let message = error.to_string();
|
||||
let mut app_error = AppError::new(code, message, retryable);
|
||||
if let Some(path) = error.path() {
|
||||
app_error = app_error.with_detail(
|
||||
"path",
|
||||
serde_json::to_value(TaggedPath::from_path(path)).unwrap_or(Value::Null),
|
||||
);
|
||||
}
|
||||
app_error
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used)]
|
||||
mod tests {
|
||||
use std::{ffi::OsString, fs, str::FromStr as _};
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::execute_cleanup;
|
||||
use crate::{
|
||||
cli::CleanArgs,
|
||||
domain::Digest,
|
||||
operation::CleanupMode,
|
||||
store::{AnalysisStore, StoreEnvironment, resolve_store},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn dry_run_command_reports_without_deleting() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(OsString::from(temp.path().join("store"))),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
let store = AnalysisStore::initialize(resolve_store(None, &environment).expect("resolve"))
|
||||
.expect("store");
|
||||
let sample = Digest::from_str(&"a".repeat(64)).expect("digest");
|
||||
let profile = Digest::from_str(&"b".repeat(64)).expect("digest");
|
||||
let directory = store
|
||||
.root()
|
||||
.join("artifacts")
|
||||
.join(sample.as_str())
|
||||
.join(profile.as_str());
|
||||
fs::create_dir_all(&directory).expect("dirs");
|
||||
fs::write(directory.join(format!("{}.json", "c".repeat(64))), b"{}\n").expect("artifact");
|
||||
let args = CleanArgs {
|
||||
sample: None,
|
||||
digest: Some(sample),
|
||||
all: false,
|
||||
dry_run: true,
|
||||
yes: false,
|
||||
};
|
||||
let data = execute_cleanup(&store, &args, 1024).expect("clean");
|
||||
assert_eq!(data.mode, CleanupMode::DryRun);
|
||||
assert_eq!(data.matched.artifacts, 1);
|
||||
assert!(directory.exists());
|
||||
}
|
||||
}
|
||||
96
src/store/error.rs
Normal file
96
src/store/error.rs
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
use std::{io, path::PathBuf};
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// Stable store-layer failure classification used by command integration.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum StoreErrorKind {
|
||||
/// A configured store path violates the store path contract.
|
||||
InvalidStorePath,
|
||||
/// A path inside tool-owned storage is a symlink or otherwise unsafe.
|
||||
UnsafeStorePath,
|
||||
/// The store layout or an immutable object is corrupt.
|
||||
CorruptStore,
|
||||
/// The Sample is absent.
|
||||
SampleNotFound,
|
||||
/// The Sample could not be read.
|
||||
SampleUnreadable,
|
||||
/// The opened Sample is not a regular file.
|
||||
InvalidSampleType,
|
||||
/// The Sample exceeds the configured streaming bound.
|
||||
SampleTooLarge,
|
||||
/// The source changed while the private snapshot was copied.
|
||||
SampleChanged,
|
||||
/// The store filesystem does not have the required reserve.
|
||||
InsufficientStoreSpace,
|
||||
/// Another process owns the requested Analysis lock.
|
||||
AnalysisBusy,
|
||||
/// An immutable destination already exists with different content.
|
||||
ImmutableConflict,
|
||||
/// A cleanup transaction was staged but could not be fully deleted.
|
||||
CleanupIncomplete,
|
||||
/// General filesystem or serialization failure.
|
||||
Io,
|
||||
}
|
||||
|
||||
/// Detailed synchronous store failure.
|
||||
#[derive(Debug, Error)]
|
||||
#[error("{message}")]
|
||||
pub struct StoreError {
|
||||
kind: StoreErrorKind,
|
||||
message: String,
|
||||
path: Option<PathBuf>,
|
||||
source: Option<io::Error>,
|
||||
}
|
||||
|
||||
impl StoreError {
|
||||
pub(crate) fn new(kind: StoreErrorKind, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
message: message.into(),
|
||||
path: None,
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn at(
|
||||
kind: StoreErrorKind,
|
||||
path: impl Into<PathBuf>,
|
||||
message: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
message: message.into(),
|
||||
path: Some(path.into()),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn io(path: impl Into<PathBuf>, source: io::Error) -> Self {
|
||||
let path = path.into();
|
||||
Self {
|
||||
kind: StoreErrorKind::Io,
|
||||
message: format!("filesystem operation failed for {}", path.display()),
|
||||
path: Some(path),
|
||||
source: Some(source),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the stable failure class.
|
||||
#[must_use]
|
||||
pub const fn kind(&self) -> StoreErrorKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
/// Returns the exact implicated path when available.
|
||||
#[must_use]
|
||||
pub fn path(&self) -> Option<&std::path::Path> {
|
||||
self.path.as_deref()
|
||||
}
|
||||
|
||||
/// Returns the underlying I/O error when one was retained.
|
||||
#[must_use]
|
||||
pub const fn io_source(&self) -> Option<&io::Error> {
|
||||
self.source.as_ref()
|
||||
}
|
||||
}
|
||||
332
src/store/filesystem.rs
Normal file
332
src/store/filesystem.rs
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
use std::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::Write as _,
|
||||
os::unix::fs::{MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _},
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::operation::{StorageUsage, StoreSource};
|
||||
|
||||
use super::{DIRECTORY_MODE, FILE_MODE, ResolvedStore, StoreError, StoreErrorKind};
|
||||
|
||||
/// Current on-disk layout version recorded in `store.json`.
|
||||
pub const STORE_LAYOUT_VERSION: u32 = 1;
|
||||
|
||||
const LAYOUT_DIRECTORIES: &[&str] = &[
|
||||
"analyses",
|
||||
"artifacts",
|
||||
"diagnostics",
|
||||
"quarantine",
|
||||
"locks",
|
||||
"staging",
|
||||
"cleanup",
|
||||
];
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct StoreManifest {
|
||||
store_layout_version: u32,
|
||||
}
|
||||
|
||||
/// Initialized secure Analysis Store.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AnalysisStore {
|
||||
root: PathBuf,
|
||||
source: StoreSource,
|
||||
}
|
||||
|
||||
impl AnalysisStore {
|
||||
/// Creates or validates the version 1 layout with private permissions.
|
||||
pub fn initialize(resolved: ResolvedStore) -> Result<Self, StoreError> {
|
||||
create_private_directory(resolved.path())?;
|
||||
for component in LAYOUT_DIRECTORIES {
|
||||
create_private_directory(&resolved.path().join(component))?;
|
||||
}
|
||||
let manifest_path = resolved.path().join("store.json");
|
||||
if manifest_path.exists() {
|
||||
reject_symlink(&manifest_path)?;
|
||||
let bytes =
|
||||
fs::read(&manifest_path).map_err(|error| StoreError::io(&manifest_path, error))?;
|
||||
let manifest: StoreManifest = serde_json::from_slice(&bytes).map_err(|error| {
|
||||
StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&manifest_path,
|
||||
format!("store.json is invalid: {error}"),
|
||||
)
|
||||
})?;
|
||||
if manifest.store_layout_version != STORE_LAYOUT_VERSION {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
&manifest_path,
|
||||
"unsupported Analysis Store layout version",
|
||||
));
|
||||
}
|
||||
set_file_mode(&manifest_path, FILE_MODE)?;
|
||||
} else {
|
||||
let manifest = StoreManifest {
|
||||
store_layout_version: STORE_LAYOUT_VERSION,
|
||||
};
|
||||
atomic_json_write(resolved.path(), &manifest_path, &manifest)?;
|
||||
}
|
||||
Ok(Self {
|
||||
root: resolved.path().to_path_buf(),
|
||||
source: resolved.source(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the absolute store root.
|
||||
#[must_use]
|
||||
pub fn root(&self) -> &Path {
|
||||
&self.root
|
||||
}
|
||||
|
||||
/// Returns the resolution precedence source.
|
||||
#[must_use]
|
||||
pub const fn source(&self) -> StoreSource {
|
||||
self.source
|
||||
}
|
||||
|
||||
/// Scans with `lstat`, rejecting symlinks and special files.
|
||||
pub fn usage(&self) -> Result<StoreUsage, StoreError> {
|
||||
scan_usage(&self.root)
|
||||
}
|
||||
|
||||
pub(crate) fn path(&self, component: &str) -> PathBuf {
|
||||
self.root.join(component)
|
||||
}
|
||||
}
|
||||
|
||||
/// Logical file bytes and allocated blocks for a traversed tree.
|
||||
pub type StoreUsage = StorageUsage;
|
||||
|
||||
pub(crate) fn create_private_directory(path: &Path) -> Result<(), StoreError> {
|
||||
reject_symlink_ancestors(path)?;
|
||||
match fs::symlink_metadata(path) {
|
||||
Ok(metadata) => {
|
||||
if !metadata.file_type().is_dir() {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::UnsafeStorePath,
|
||||
path,
|
||||
"tool-owned path is not a directory",
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
fs::create_dir_all(path).map_err(|error| StoreError::io(path, error))?;
|
||||
reject_symlink(path)?;
|
||||
}
|
||||
Err(error) => return Err(StoreError::io(path, error)),
|
||||
}
|
||||
fs::set_permissions(path, fs::Permissions::from_mode(DIRECTORY_MODE))
|
||||
.map_err(|error| StoreError::io(path, error))
|
||||
}
|
||||
|
||||
fn reject_symlink_ancestors(path: &Path) -> Result<(), StoreError> {
|
||||
for ancestor in path.ancestors().collect::<Vec<_>>().into_iter().rev() {
|
||||
match fs::symlink_metadata(ancestor) {
|
||||
Ok(metadata) if metadata.file_type().is_symlink() => {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::UnsafeStorePath,
|
||||
ancestor,
|
||||
"symlink found in tool-owned path traversal",
|
||||
));
|
||||
}
|
||||
Ok(metadata) if ancestor != path && !metadata.is_dir() => {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::UnsafeStorePath,
|
||||
ancestor,
|
||||
"non-directory found in tool-owned path traversal",
|
||||
));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(StoreError::io(ancestor, error)),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn reject_symlink(path: &Path) -> Result<(), StoreError> {
|
||||
let metadata = fs::symlink_metadata(path).map_err(|error| StoreError::io(path, error))?;
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::UnsafeStorePath,
|
||||
path,
|
||||
"symlink found inside tool-owned storage",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn set_file_mode(path: &Path, mode: u32) -> Result<(), StoreError> {
|
||||
fs::set_permissions(path, fs::Permissions::from_mode(mode))
|
||||
.map_err(|error| StoreError::io(path, error))
|
||||
}
|
||||
|
||||
pub(crate) fn open_new_private_file(path: &Path) -> Result<File, StoreError> {
|
||||
OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(FILE_MODE)
|
||||
.open(path)
|
||||
.map_err(|error| StoreError::io(path, error))
|
||||
}
|
||||
|
||||
pub(crate) fn atomic_json_write<T: Serialize>(
|
||||
temporary_parent: &Path,
|
||||
destination: &Path,
|
||||
value: &T,
|
||||
) -> Result<(), StoreError> {
|
||||
let temporary = temporary_parent.join(format!("manifest-{}.tmp", random_id()?));
|
||||
let mut bytes = serde_json::to_vec(value).map_err(|error| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::Io,
|
||||
format!("JSON serialization failed: {error}"),
|
||||
)
|
||||
})?;
|
||||
bytes.push(b'\n');
|
||||
let result = (|| {
|
||||
let mut file = open_new_private_file(&temporary)?;
|
||||
file.write_all(&bytes)
|
||||
.map_err(|error| StoreError::io(&temporary, error))?;
|
||||
file.sync_all()
|
||||
.map_err(|error| StoreError::io(&temporary, error))?;
|
||||
fs::rename(&temporary, destination).map_err(|error| StoreError::io(destination, error))?;
|
||||
sync_directory(temporary_parent)
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(&temporary);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) fn sync_directory(path: &Path) -> Result<(), StoreError> {
|
||||
File::open(path)
|
||||
.and_then(|file| file.sync_all())
|
||||
.map_err(|error| StoreError::io(path, error))
|
||||
}
|
||||
|
||||
pub(crate) fn random_id() -> Result<String, StoreError> {
|
||||
let mut bytes = [0_u8; 16];
|
||||
getrandom::fill(&mut bytes).map_err(|error| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::Io,
|
||||
format!("random ID generation failed: {error}"),
|
||||
)
|
||||
})?;
|
||||
Ok(hex::encode(bytes))
|
||||
}
|
||||
|
||||
pub(crate) fn scan_usage(path: &Path) -> Result<StorageUsage, StoreError> {
|
||||
let metadata = fs::symlink_metadata(path).map_err(|error| StoreError::io(path, error))?;
|
||||
let file_type = metadata.file_type();
|
||||
if file_type.is_symlink() {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::UnsafeStorePath,
|
||||
path,
|
||||
"symlink found while scanning tool-owned storage",
|
||||
));
|
||||
}
|
||||
if !(file_type.is_file() || file_type.is_dir()) {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::CorruptStore,
|
||||
path,
|
||||
"special file found while scanning tool-owned storage",
|
||||
));
|
||||
}
|
||||
let mut usage = StorageUsage {
|
||||
logical_bytes: if file_type.is_file() {
|
||||
metadata.len()
|
||||
} else {
|
||||
0
|
||||
},
|
||||
allocated_bytes: metadata.blocks().saturating_mul(512),
|
||||
};
|
||||
if file_type.is_dir() {
|
||||
let mut entries = fs::read_dir(path)
|
||||
.map_err(|error| StoreError::io(path, error))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|error| StoreError::io(path, error))?;
|
||||
entries.sort_by_key(std::fs::DirEntry::file_name);
|
||||
for entry in entries {
|
||||
let child = scan_usage(&entry.path())?;
|
||||
usage.logical_bytes = usage
|
||||
.logical_bytes
|
||||
.checked_add(child.logical_bytes)
|
||||
.ok_or_else(|| {
|
||||
StoreError::new(StoreErrorKind::CorruptStore, "logical usage overflow")
|
||||
})?;
|
||||
usage.allocated_bytes = usage
|
||||
.allocated_bytes
|
||||
.checked_add(child.allocated_bytes)
|
||||
.ok_or_else(|| {
|
||||
StoreError::new(StoreErrorKind::CorruptStore, "allocated usage overflow")
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(usage)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used)]
|
||||
mod tests {
|
||||
use std::{
|
||||
ffi::OsString,
|
||||
fs,
|
||||
os::unix::fs::{MetadataExt as _, PermissionsExt as _, symlink},
|
||||
};
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::AnalysisStore;
|
||||
use crate::store::{StoreEnvironment, resolve_store};
|
||||
|
||||
fn store(temp: &TempDir) -> AnalysisStore {
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(OsString::from(temp.path().join("store"))),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
AnalysisStore::initialize(resolve_store(None, &environment).expect("resolve"))
|
||||
.expect("initialize")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn initialization_is_private_and_versioned() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let store = store(&temp);
|
||||
assert_eq!(
|
||||
fs::metadata(store.root())
|
||||
.expect("metadata")
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777,
|
||||
0o700
|
||||
);
|
||||
let manifest = fs::read_to_string(store.root().join("store.json")).expect("manifest");
|
||||
assert_eq!(manifest, "{\"store_layout_version\":1}\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_counts_logical_and_allocated_bytes() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let store = store(&temp);
|
||||
fs::write(store.root().join("artifacts/file"), b"abc").expect("write");
|
||||
let usage = store.usage().expect("usage");
|
||||
assert!(usage.logical_bytes >= 3);
|
||||
assert!(
|
||||
usage.allocated_bytes >= fs::metadata(store.root()).expect("metadata").blocks() * 512
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_scan_never_follows_symlinks() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let store = store(&temp);
|
||||
let outside = temp.path().join("outside");
|
||||
fs::write(&outside, vec![0_u8; 1024]).expect("outside");
|
||||
symlink(&outside, store.root().join("artifacts/escape")).expect("symlink");
|
||||
assert!(store.usage().is_err());
|
||||
}
|
||||
}
|
||||
112
src/store/lock.rs
Normal file
112
src/store/lock.rs
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
use std::{
|
||||
fs::File,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use rustix::fs::{FlockOperation, Mode, OFlags};
|
||||
|
||||
use crate::domain::Digest;
|
||||
|
||||
use super::{AnalysisStore, StoreError, StoreErrorKind, filesystem::create_private_directory};
|
||||
|
||||
/// Held nonblocking exclusive per-Analysis OS lock.
|
||||
#[derive(Debug)]
|
||||
pub struct AnalysisLock {
|
||||
_file: File,
|
||||
path: PathBuf,
|
||||
sample: Digest,
|
||||
profile: Digest,
|
||||
}
|
||||
|
||||
impl AnalysisLock {
|
||||
/// Immediately acquires `<sample>/<profile>.lock`, or reports `AnalysisBusy`.
|
||||
pub fn acquire(
|
||||
store: &AnalysisStore,
|
||||
sample: &Digest,
|
||||
profile: &Digest,
|
||||
) -> Result<Self, StoreError> {
|
||||
let directory = store.path("locks").join(sample.as_str());
|
||||
create_private_directory(&directory)?;
|
||||
let path = directory.join(format!("{}.lock", profile.as_str()));
|
||||
let descriptor = rustix::fs::open(
|
||||
&path,
|
||||
OFlags::RDWR | OFlags::CREATE | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)
|
||||
.map_err(|error| {
|
||||
StoreError::at(
|
||||
StoreErrorKind::UnsafeStorePath,
|
||||
&path,
|
||||
format!("failed to safely open Analysis lock: {error}"),
|
||||
)
|
||||
})?;
|
||||
let file = File::from(descriptor);
|
||||
match rustix::fs::flock(&file, FlockOperation::NonBlockingLockExclusive) {
|
||||
Ok(()) => Ok(Self {
|
||||
_file: file,
|
||||
path,
|
||||
sample: sample.clone(),
|
||||
profile: profile.clone(),
|
||||
}),
|
||||
Err(error) if error == rustix::io::Errno::WOULDBLOCK => Err(StoreError::at(
|
||||
StoreErrorKind::AnalysisBusy,
|
||||
&path,
|
||||
format!("Analysis {} is locked by another process", profile.as_str()),
|
||||
)),
|
||||
Err(error) => Err(StoreError::at(
|
||||
StoreErrorKind::Io,
|
||||
&path,
|
||||
format!("failed to lock Analysis: {error}"),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the persistent lock-file path.
|
||||
#[must_use]
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Returns the locked Analysis Profile digest.
|
||||
#[must_use]
|
||||
pub const fn profile(&self) -> &Digest {
|
||||
&self.profile
|
||||
}
|
||||
|
||||
/// Returns whether this token guards the exact requested Analysis identity.
|
||||
#[must_use]
|
||||
pub fn guards(&self, sample: &Digest, profile: &Digest) -> bool {
|
||||
&self.sample == sample && &self.profile == profile
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used)]
|
||||
mod tests {
|
||||
use std::{ffi::OsString, str::FromStr as _};
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::AnalysisLock;
|
||||
use crate::{
|
||||
domain::Digest,
|
||||
store::{AnalysisStore, StoreEnvironment, resolve_store},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn second_process_style_lock_attempt_fails_immediately() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(OsString::from(temp.path().join("store"))),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
let store = AnalysisStore::initialize(resolve_store(None, &environment).expect("resolve"))
|
||||
.expect("store");
|
||||
let sample = Digest::from_str(&"a".repeat(64)).expect("digest");
|
||||
let profile = Digest::from_str(&"b".repeat(64)).expect("digest");
|
||||
let first = AnalysisLock::acquire(&store, &sample, &profile).expect("first");
|
||||
assert!(AnalysisLock::acquire(&store, &sample, &profile).is_err());
|
||||
drop(first);
|
||||
assert!(AnalysisLock::acquire(&store, &sample, &profile).is_ok());
|
||||
}
|
||||
}
|
||||
28
src/store/mod.rs
Normal file
28
src/store/mod.rs
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
//! Secure, synchronous Sample snapshot and Analysis Store primitives.
|
||||
|
||||
mod analysis;
|
||||
mod artifact;
|
||||
mod cleanup;
|
||||
mod cleanup_command;
|
||||
mod error;
|
||||
mod filesystem;
|
||||
mod lock;
|
||||
mod resolution;
|
||||
mod sample;
|
||||
|
||||
pub use analysis::{
|
||||
AnalysisManifest, AnalysisProfileManifest, AnalysisProject, CreatedBy, InventoryFile,
|
||||
ManifestSample, QuarantineOutcome, RebuildDecision, analysis_profile_digest, inventory_project,
|
||||
rebuild_decision, validate_analysis, write_analysis_manifest,
|
||||
};
|
||||
pub use artifact::{Artifact, publish_artifact};
|
||||
pub use cleanup::{CleanupPlan, CleanupScope, CleanupTransaction, plan_cleanup};
|
||||
pub use cleanup_command::execute_cleanup;
|
||||
pub use error::{StoreError, StoreErrorKind};
|
||||
pub use filesystem::{AnalysisStore, STORE_LAYOUT_VERSION, StoreUsage};
|
||||
pub use lock::AnalysisLock;
|
||||
pub use resolution::{ResolvedStore, StoreEnvironment, resolve_store};
|
||||
pub use sample::{DEFAULT_STORE_RESERVE_BYTES, SampleSnapshot, SnapshotOptions, snapshot_sample};
|
||||
|
||||
pub(crate) const DIRECTORY_MODE: u32 = 0o700;
|
||||
pub(crate) const FILE_MODE: u32 = 0o600;
|
||||
166
src/store/resolution.rs
Normal file
166
src/store/resolution.rs
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
use std::{
|
||||
ffi::OsString,
|
||||
path::{Component, Path, PathBuf},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
domain::TaggedPath,
|
||||
operation::{AnalysisStoreProvenance, StoreSource},
|
||||
};
|
||||
|
||||
use super::{StoreError, StoreErrorKind};
|
||||
|
||||
/// Environment values that participate in deterministic store resolution.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct StoreEnvironment {
|
||||
/// Exact `GHIDR_STORE` value, when set.
|
||||
pub ghidr_store: Option<OsString>,
|
||||
/// Exact `XDG_CACHE_HOME` value, when explicitly set.
|
||||
pub xdg_cache_home: Option<OsString>,
|
||||
/// Exact home directory used only for the final fallback.
|
||||
pub home: Option<OsString>,
|
||||
}
|
||||
|
||||
impl StoreEnvironment {
|
||||
/// Captures the three relevant variables without lossy decoding.
|
||||
#[must_use]
|
||||
pub fn from_process() -> Self {
|
||||
Self {
|
||||
ghidr_store: std::env::var_os("GHIDR_STORE"),
|
||||
xdg_cache_home: std::env::var_os("XDG_CACHE_HOME"),
|
||||
home: std::env::var_os("HOME"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A validated absolute UTF-8 store path and its precedence source.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ResolvedStore {
|
||||
path: PathBuf,
|
||||
source: StoreSource,
|
||||
}
|
||||
|
||||
impl ResolvedStore {
|
||||
/// Returns the absolute UTF-8 store path.
|
||||
#[must_use]
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Returns the selected precedence source.
|
||||
#[must_use]
|
||||
pub const fn source(&self) -> StoreSource {
|
||||
self.source
|
||||
}
|
||||
|
||||
/// Returns the public exact-byte path representation.
|
||||
#[must_use]
|
||||
pub fn tagged_path(&self) -> TaggedPath {
|
||||
TaggedPath::from_path(&self.path)
|
||||
}
|
||||
|
||||
/// Builds the fixed public provenance object for this resolution.
|
||||
#[must_use]
|
||||
pub fn provenance(&self) -> AnalysisStoreProvenance {
|
||||
AnalysisStoreProvenance {
|
||||
path: self.tagged_path(),
|
||||
source: self.source,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the store using CLI, `GHIDR_STORE`, explicit XDG, then home precedence.
|
||||
pub fn resolve_store(
|
||||
cli_path: Option<&Path>,
|
||||
environment: &StoreEnvironment,
|
||||
) -> Result<ResolvedStore, StoreError> {
|
||||
let (path, source) = if let Some(path) = cli_path {
|
||||
(path.to_path_buf(), StoreSource::Cli)
|
||||
} else if let Some(path) = &environment.ghidr_store {
|
||||
(PathBuf::from(path), StoreSource::Environment)
|
||||
} else if let Some(path) = &environment.xdg_cache_home {
|
||||
(PathBuf::from(path).join("ghidr"), StoreSource::Xdg)
|
||||
} else if let Some(path) = &environment.home {
|
||||
(
|
||||
PathBuf::from(path).join("ghidr-store"),
|
||||
StoreSource::HomeFallback,
|
||||
)
|
||||
} else {
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::InvalidStorePath,
|
||||
"no Analysis Store path is available: HOME is not set",
|
||||
));
|
||||
};
|
||||
validate_store_path(&path)?;
|
||||
Ok(ResolvedStore { path, source })
|
||||
}
|
||||
|
||||
fn validate_store_path(path: &Path) -> Result<(), StoreError> {
|
||||
let Some(text) = path.to_str() else {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::InvalidStorePath,
|
||||
path,
|
||||
"Analysis Store path must be valid UTF-8",
|
||||
));
|
||||
};
|
||||
if !path.is_absolute()
|
||||
|| text.contains(['\n', '\r'])
|
||||
|| path
|
||||
.components()
|
||||
.any(|component| matches!(component, Component::CurDir | Component::ParentDir))
|
||||
{
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::InvalidStorePath,
|
||||
path,
|
||||
"Analysis Store path must be absolute, normalized UTF-8",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used)]
|
||||
mod tests {
|
||||
use std::{ffi::OsString, path::Path};
|
||||
|
||||
use super::{StoreEnvironment, resolve_store};
|
||||
use crate::operation::StoreSource;
|
||||
|
||||
#[test]
|
||||
fn resolution_obeys_fixed_precedence() {
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(OsString::from("/environment")),
|
||||
xdg_cache_home: Some(OsString::from("/xdg")),
|
||||
home: Some(OsString::from("/home/user")),
|
||||
};
|
||||
let resolved = resolve_store(Some(Path::new("/cli")), &environment).expect("valid");
|
||||
assert_eq!(resolved.path(), Path::new("/cli"));
|
||||
assert_eq!(resolved.source(), StoreSource::Cli);
|
||||
|
||||
let resolved = resolve_store(None, &environment).expect("valid");
|
||||
assert_eq!(resolved.path(), Path::new("/environment"));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rejects_exact_non_utf8_configured_path() {
|
||||
use std::os::unix::ffi::OsStringExt as _;
|
||||
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(OsString::from_vec(vec![b'/', b'x', 0xff])),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
let error = resolve_store(None, &environment).expect_err("must reject");
|
||||
assert!(error.path().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_explicit_xdg_does_not_fall_back() {
|
||||
let environment = StoreEnvironment {
|
||||
xdg_cache_home: Some(OsString::from("relative")),
|
||||
home: Some(OsString::from("/home/user")),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
assert!(resolve_store(None, &environment).is_err());
|
||||
}
|
||||
}
|
||||
380
src/store/sample.rs
Normal file
380
src/store/sample.rs
Normal file
|
|
@ -0,0 +1,380 @@
|
|||
use std::{
|
||||
fs::{self, File},
|
||||
io::{Read as _, Write as _},
|
||||
os::unix::fs::{FileTypeExt as _, MetadataExt as _},
|
||||
path::{Path, PathBuf},
|
||||
str::FromStr as _,
|
||||
};
|
||||
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
use crate::domain::{Digest, TaggedPath};
|
||||
|
||||
use super::{
|
||||
AnalysisStore, StoreError, StoreErrorKind,
|
||||
filesystem::{create_private_directory, open_new_private_file, random_id, set_file_mode},
|
||||
};
|
||||
|
||||
/// Default free-space reserve retained beyond the observed Sample size.
|
||||
pub const DEFAULT_STORE_RESERVE_BYTES: u64 = 1_073_741_824;
|
||||
|
||||
/// Bounds governing one staged Sample copy.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct SnapshotOptions {
|
||||
/// Maximum bytes accepted before and during streaming.
|
||||
pub max_sample_bytes: u64,
|
||||
/// Bytes that must remain available beyond the observed Sample size.
|
||||
pub reserve_bytes: u64,
|
||||
}
|
||||
|
||||
impl SnapshotOptions {
|
||||
/// Constructs the production reserve policy for a positive byte ceiling.
|
||||
pub fn new(max_sample_bytes: u64) -> Result<Self, StoreError> {
|
||||
if max_sample_bytes == 0 {
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::SampleTooLarge,
|
||||
"maximum Sample size must be positive",
|
||||
));
|
||||
}
|
||||
Ok(Self {
|
||||
max_sample_bytes,
|
||||
reserve_bytes: DEFAULT_STORE_RESERVE_BYTES,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Private read-only Sample snapshot and its content identity.
|
||||
#[derive(Debug)]
|
||||
pub struct SampleSnapshot {
|
||||
digest: Digest,
|
||||
size_bytes: u64,
|
||||
path: PathBuf,
|
||||
staging_directory: PathBuf,
|
||||
source_path: TaggedPath,
|
||||
}
|
||||
|
||||
impl SampleSnapshot {
|
||||
/// SHA-256 of the exact staged bytes.
|
||||
#[must_use]
|
||||
pub const fn digest(&self) -> &Digest {
|
||||
&self.digest
|
||||
}
|
||||
|
||||
/// Exact staged byte count.
|
||||
#[must_use]
|
||||
pub const fn size_bytes(&self) -> u64 {
|
||||
self.size_bytes
|
||||
}
|
||||
|
||||
/// Tool-generated UTF-8 path that may be passed to the worker.
|
||||
#[must_use]
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Exact caller-supplied path retained only as provenance.
|
||||
#[must_use]
|
||||
pub const fn source_path(&self) -> &TaggedPath {
|
||||
&self.source_path
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SampleSnapshot {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.staging_directory);
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens, bounds, streams, hashes, verifies, and privately stages one Sample.
|
||||
pub fn snapshot_sample(
|
||||
store: &AnalysisStore,
|
||||
source: &Path,
|
||||
options: SnapshotOptions,
|
||||
) -> Result<SampleSnapshot, StoreError> {
|
||||
snapshot_with_hook(store, source, options, |_| {})
|
||||
}
|
||||
|
||||
fn snapshot_with_hook(
|
||||
store: &AnalysisStore,
|
||||
source: &Path,
|
||||
options: SnapshotOptions,
|
||||
mut after_chunk: impl FnMut(u64),
|
||||
) -> Result<SampleSnapshot, StoreError> {
|
||||
if options.max_sample_bytes == 0 {
|
||||
return Err(StoreError::new(
|
||||
StoreErrorKind::SampleTooLarge,
|
||||
"maximum Sample size must be positive",
|
||||
));
|
||||
}
|
||||
let mut input = File::open(source).map_err(|error| sample_open_error(source, error))?;
|
||||
let before = input
|
||||
.metadata()
|
||||
.map_err(|error| sample_read_error(source, error))?;
|
||||
if !before.file_type().is_file()
|
||||
|| before.file_type().is_dir()
|
||||
|| before.file_type().is_fifo()
|
||||
|| before.file_type().is_socket()
|
||||
|| before.file_type().is_block_device()
|
||||
|| before.file_type().is_char_device()
|
||||
{
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::InvalidSampleType,
|
||||
source,
|
||||
"opened Sample is not a regular file",
|
||||
));
|
||||
}
|
||||
if before.len() > options.max_sample_bytes {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::SampleTooLarge,
|
||||
source,
|
||||
"Sample exceeds the configured byte limit",
|
||||
));
|
||||
}
|
||||
require_free_space(store.root(), before.len(), options.reserve_bytes)?;
|
||||
|
||||
let staging_directory = store.path("staging").join(random_id()?);
|
||||
create_private_directory(&staging_directory)?;
|
||||
let snapshot_path = staging_directory.join("sample");
|
||||
let result = (|| {
|
||||
let mut output = open_new_private_file(&snapshot_path)?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut size_bytes = 0_u64;
|
||||
let mut buffer = vec![0_u8; 1024 * 1024];
|
||||
loop {
|
||||
let count = input
|
||||
.read(&mut buffer)
|
||||
.map_err(|error| sample_read_error(source, error))?;
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
size_bytes = size_bytes
|
||||
.checked_add(u64::try_from(count).map_err(|_| {
|
||||
StoreError::new(StoreErrorKind::SampleTooLarge, "Sample byte count overflow")
|
||||
})?)
|
||||
.ok_or_else(|| {
|
||||
StoreError::new(StoreErrorKind::SampleTooLarge, "Sample byte count overflow")
|
||||
})?;
|
||||
if size_bytes > options.max_sample_bytes {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::SampleTooLarge,
|
||||
source,
|
||||
"Sample grew beyond the configured byte limit while streaming",
|
||||
));
|
||||
}
|
||||
output
|
||||
.write_all(&buffer[..count])
|
||||
.map_err(|error| StoreError::io(&snapshot_path, error))?;
|
||||
hasher.update(&buffer[..count]);
|
||||
after_chunk(size_bytes);
|
||||
}
|
||||
output
|
||||
.sync_all()
|
||||
.map_err(|error| StoreError::io(&snapshot_path, error))?;
|
||||
set_file_mode(&snapshot_path, 0o400)?;
|
||||
let after = input
|
||||
.metadata()
|
||||
.map_err(|error| sample_read_error(source, error))?;
|
||||
if source_metadata_changed(&before, &after) || after.len() != size_bytes {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::SampleChanged,
|
||||
source,
|
||||
"Sample metadata changed while creating the private snapshot",
|
||||
));
|
||||
}
|
||||
let digest = Digest::from_str(&hex::encode(hasher.finalize())).map_err(|error| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::Io,
|
||||
format!("computed digest was invalid: {error}"),
|
||||
)
|
||||
})?;
|
||||
Ok(SampleSnapshot {
|
||||
digest,
|
||||
size_bytes,
|
||||
path: snapshot_path.clone(),
|
||||
staging_directory: staging_directory.clone(),
|
||||
source_path: TaggedPath::from_path(source),
|
||||
})
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_dir_all(&staging_directory);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn require_free_space(path: &Path, observed: u64, reserve: u64) -> Result<(), StoreError> {
|
||||
let required = observed.checked_add(reserve).ok_or_else(|| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::InsufficientStoreSpace,
|
||||
"required free-space bound overflow",
|
||||
)
|
||||
})?;
|
||||
let stats = rustix::fs::statvfs(path).map_err(|error| {
|
||||
StoreError::new(
|
||||
StoreErrorKind::Io,
|
||||
format!("statvfs failed for {}: {error}", path.display()),
|
||||
)
|
||||
})?;
|
||||
let available = stats.f_bavail.saturating_mul(stats.f_frsize);
|
||||
if available < required {
|
||||
return Err(StoreError::at(
|
||||
StoreErrorKind::InsufficientStoreSpace,
|
||||
path,
|
||||
format!("Analysis Store has {available} bytes available but {required} are required"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn source_metadata_changed(before: &fs::Metadata, after: &fs::Metadata) -> bool {
|
||||
before.dev() != after.dev()
|
||||
|| before.ino() != after.ino()
|
||||
|| before.len() != after.len()
|
||||
|| before.mtime() != after.mtime()
|
||||
|| before.mtime_nsec() != after.mtime_nsec()
|
||||
|| before.ctime() != after.ctime()
|
||||
|| before.ctime_nsec() != after.ctime_nsec()
|
||||
}
|
||||
|
||||
fn sample_open_error(path: &Path, error: std::io::Error) -> StoreError {
|
||||
let kind = if error.kind() == std::io::ErrorKind::NotFound {
|
||||
StoreErrorKind::SampleNotFound
|
||||
} else {
|
||||
StoreErrorKind::SampleUnreadable
|
||||
};
|
||||
StoreError::at(kind, path, format!("failed to open Sample: {error}"))
|
||||
}
|
||||
|
||||
fn sample_read_error(path: &Path, error: std::io::Error) -> StoreError {
|
||||
StoreError::at(
|
||||
StoreErrorKind::SampleUnreadable,
|
||||
path,
|
||||
format!("failed to read Sample: {error}"),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used)]
|
||||
mod tests {
|
||||
use std::{
|
||||
ffi::OsString, fs, os::unix::ffi::OsStringExt as _, os::unix::fs::PermissionsExt as _,
|
||||
};
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::{SnapshotOptions, snapshot_sample, snapshot_with_hook};
|
||||
use crate::{
|
||||
domain::TaggedPath,
|
||||
store::{AnalysisStore, StoreEnvironment, resolve_store},
|
||||
};
|
||||
|
||||
fn store(temp: &TempDir) -> AnalysisStore {
|
||||
let environment = StoreEnvironment {
|
||||
ghidr_store: Some(temp.path().join("store").into_os_string()),
|
||||
..StoreEnvironment::default()
|
||||
};
|
||||
AnalysisStore::initialize(resolve_store(None, &environment).expect("resolve"))
|
||||
.expect("store")
|
||||
}
|
||||
|
||||
fn options() -> SnapshotOptions {
|
||||
SnapshotOptions {
|
||||
max_sample_bytes: 1024 * 1024,
|
||||
reserve_bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_bytes_have_identical_identity_and_read_only_snapshots() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let store = store(&temp);
|
||||
let first = temp.path().join("first");
|
||||
let second = temp.path().join("second");
|
||||
fs::write(&first, b"same bytes").expect("first");
|
||||
fs::write(&second, b"same bytes").expect("second");
|
||||
let first_snapshot = snapshot_sample(&store, &first, options()).expect("snapshot");
|
||||
let second_snapshot = snapshot_sample(&store, &second, options()).expect("snapshot");
|
||||
assert_eq!(first_snapshot.digest(), second_snapshot.digest());
|
||||
assert_eq!(
|
||||
fs::metadata(first_snapshot.path())
|
||||
.expect("metadata")
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777,
|
||||
0o400
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_bound_rejects_oversized_sample() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let store = store(&temp);
|
||||
let sample = temp.path().join("large");
|
||||
fs::write(&sample, b"12345").expect("write");
|
||||
assert!(
|
||||
snapshot_sample(
|
||||
&store,
|
||||
&sample,
|
||||
SnapshotOptions {
|
||||
max_sample_bytes: 4,
|
||||
reserve_bytes: 0
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_source_is_rejected_and_staging_is_removed() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let store = store(&temp);
|
||||
let sample = temp.path().join("changing");
|
||||
fs::write(&sample, vec![b'a'; 16]).expect("write");
|
||||
let mut changed = false;
|
||||
let result = snapshot_with_hook(&store, &sample, options(), |_| {
|
||||
if !changed {
|
||||
fs::write(&sample, vec![b'b'; 17]).expect("change");
|
||||
changed = true;
|
||||
}
|
||||
});
|
||||
assert!(result.is_err());
|
||||
assert_eq!(
|
||||
fs::read_dir(store.root().join("staging"))
|
||||
.expect("staging")
|
||||
.count(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_utf8_source_is_preserved_exactly_as_provenance() {
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let store = store(&temp);
|
||||
let sample = temp.path().join(OsString::from_vec(vec![b's', 0xff]));
|
||||
fs::write(&sample, b"bytes").expect("write");
|
||||
let snapshot = snapshot_sample(&store, &sample, options()).expect("snapshot");
|
||||
assert!(matches!(
|
||||
snapshot.source_path(),
|
||||
TaggedPath::UnixBytesBase64(_)
|
||||
));
|
||||
assert_eq!(
|
||||
snapshot.source_path().to_path_buf().expect("decode"),
|
||||
sample
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaced_staging_directory_symlink_is_never_traversed() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let temp = TempDir::new().expect("temp");
|
||||
let store = store(&temp);
|
||||
let outside = temp.path().join("outside");
|
||||
fs::create_dir(&outside).expect("outside");
|
||||
fs::remove_dir(store.root().join("staging")).expect("remove staging");
|
||||
symlink(&outside, store.root().join("staging")).expect("attack");
|
||||
let sample = temp.path().join("sample");
|
||||
fs::write(&sample, b"bytes").expect("sample");
|
||||
assert!(snapshot_sample(&store, &sample, options()).is_err());
|
||||
assert_eq!(fs::read_dir(outside).expect("outside entries").count(), 0);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue