Skip to content

Commit 8d6eaac

Browse files
committed
fix: #478 make s3 upload confirmation resilient
1 parent 3fe1365 commit 8d6eaac

2 files changed

Lines changed: 70 additions & 11 deletions

File tree

core/storage.py

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -536,18 +536,31 @@ async def file_exists(self, save_path: str) -> bool:
536536
:param save_path: 文件路径
537537
:return: 文件是否存在
538538
"""
539-
async with self.session.client(
540-
"s3",
541-
endpoint_url=self.endpoint_url,
542-
aws_session_token=self.aws_session_token,
543-
region_name=self.region_name,
544-
config=Config(signature_version=self.signature_version),
545-
) as s3:
539+
async with self._client() as s3:
540+
last_error = None
541+
for attempt in range(3):
542+
try:
543+
await s3.head_object(Bucket=self.bucket_name, Key=save_path)
544+
return True
545+
except Exception as e:
546+
last_error = e
547+
if attempt < 2:
548+
await asyncio.sleep(0.2 * (attempt + 1))
549+
546550
try:
547-
await s3.head_object(Bucket=self.bucket_name, Key=save_path)
548-
return True
549-
except Exception:
550-
return False
551+
result = await s3.list_objects_v2(
552+
Bucket=self.bucket_name,
553+
Prefix=save_path,
554+
MaxKeys=1,
555+
)
556+
for item in result.get("Contents", []):
557+
if item.get("Key") == save_path:
558+
return True
559+
except Exception as e:
560+
last_error = e
561+
562+
logger.warning(f"S3文件确认失败 key={save_path}: {last_error}")
563+
return False
551564

552565

553566
class OneDriveFileStorage(FileStorageInterface):
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import asyncio
2+
import unittest
3+
4+
from core.storage import S3FileStorage
5+
6+
7+
class FakeS3Client:
8+
def __init__(self, list_response):
9+
self.list_response = list_response
10+
self.head_calls = 0
11+
12+
async def __aenter__(self):
13+
return self
14+
15+
async def __aexit__(self, exc_type, exc, traceback):
16+
return False
17+
18+
async def head_object(self, **kwargs):
19+
self.head_calls += 1
20+
raise RuntimeError("head not supported")
21+
22+
async def list_objects_v2(self, **kwargs):
23+
return self.list_response
24+
25+
26+
class S3FileExistsTests(unittest.TestCase):
27+
def test_file_exists_falls_back_to_list_objects(self):
28+
client = FakeS3Client({"Contents": [{"Key": "share/data/file.txt"}]})
29+
storage = S3FileStorage.__new__(S3FileStorage)
30+
storage.bucket_name = "bucket"
31+
storage._client = lambda: client
32+
33+
exists = asyncio.run(storage.file_exists("share/data/file.txt"))
34+
35+
self.assertTrue(exists)
36+
self.assertEqual(client.head_calls, 3)
37+
38+
def test_file_exists_returns_false_when_missing(self):
39+
client = FakeS3Client({"Contents": [{"Key": "share/data/other.txt"}]})
40+
storage = S3FileStorage.__new__(S3FileStorage)
41+
storage.bucket_name = "bucket"
42+
storage._client = lambda: client
43+
44+
exists = asyncio.run(storage.file_exists("share/data/file.txt"))
45+
46+
self.assertFalse(exists)

0 commit comments

Comments
 (0)