Skip to content

Commit 47ab4da

Browse files
ShawnMcKeekofemann
ShawnMcKee
authored and
kofemann
committed
Add WAN SciTag support for remote HTTP(S) TPC movers
Signed-off-by: Shawn McKee <smckee@umich.edu> (cherry picked from commit 88a9be1) Signed-off-by: Tigran Mkrtchyan <tigran.mkrtchyan@desy.de>
1 parent 5fe8edf commit 47ab4da

5 files changed

Lines changed: 99 additions & 7 deletions

File tree

‎modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ public class RemoteHttpDataTransferProtocolInfo implements IpProtocolInfo {
2929
private final boolean isVerificationRequired;
3030
private final ImmutableMap<String, String> headers;
3131
private final OpenIdCredential openIdCredential;
32+
private String transferTag = "";
3233
@Nullable
3334
private final ChecksumType desiredChecksum;
3435
@Nonnull
@@ -104,6 +105,15 @@ public ImmutableMap<String, String> getHeaders() {
104105
return headers;
105106
}
106107

108+
public void setTransferTag(String transferTag) {
109+
this.transferTag = transferTag == null ? "" : transferTag;
110+
}
111+
112+
@Override
113+
public String getTransferTag() {
114+
return transferTag;
115+
}
116+
107117
@Override
108118
public String toString() {
109119
return getVersionString() + ':' + sourceHttpUrl;
@@ -136,5 +146,10 @@ private void readObject(java.io.ObjectInputStream stream)
136146
? Collections.emptySet()
137147
: Set.of(desiredChecksum);
138148
}
149+
150+
// Handle objects sent from old doors.
151+
if (transferTag == null) {
152+
transferTag = "";
153+
}
139154
}
140155
}

‎modules/dcache-webdav/src/main/java/org/dcache/webdav/LoggingHandler.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,5 +59,7 @@ protected void describeOperation(NetLoggerBuilder log,
5959
log.add("tpc.require-checksum", CopyFilter.getTpcRequireChecksumVerification(request));
6060
log.add("tpc.source", CopyFilter.getTpcSource(request));
6161
log.add("tpc.destination", CopyFilter.getTpcDestination(request));
62+
log.add("tpc.scitag", CopyFilter.getTpcSciTag(request));
63+
log.add("tpc.transferheaderscitag", CopyFilter.getTpcTransferHeaderSciTag(request));
6264
}
6365
}

‎modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/CopyFilter.java‎

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,20 @@ public class CopyFilter implements Filter {
103103

104104
private static final String QUERY_KEY_ASKED_TO_DELEGATE = "asked-to-delegate";
105105
private static final String REQUEST_HEADER_CREDENTIAL = "Credential";
106+
private static final String REQUEST_HEADER_SCITAG = "SciTag";
107+
private static final String REQUEST_HEADER_TRANSFER_HEADER_SCITAG = "TransferHeaderSciTag";
106108
private static final String REQUEST_HEADER_TRANSFER_HEADER_PREFIX = "transferheader";
107109
private static final String REQUEST_HEADER_VERIFICATION = "RequireChecksumVerification";
108110
private static final String TPC_ERROR_ATTRIBUTE = "org.dcache.tpc-error";
109111
private static final String TPC_CREDENTIAL_ATTRIBUTE = "org.dcache.tpc-credential";
110112
private static final String TPC_REQUIRE_CHECKSUM_VERIFICATION_ATTRIBUTE = "org.dcache.tpc-require-checksum-verify";
111113
private static final String TPC_SOURCE_ATTRIBUTE = "org.dcache.tpc-source";
112114
private static final String TPC_DESTINATION_ATTRIBUTE = "org.dcache.tpc-destination";
115+
private static final String TPC_SCITAG_ATTRIBUTE = "org.dcache.tpc-scitag";
116+
private static final String TPC_TRANSFER_HEADER_SCITAG_ATTRIBUTE =
117+
"org.dcache.tpc-transferheaderscitag";
118+
// Sentinel for a missing or blank SciTag-related header value.
119+
private static final String MISSING_HEADER_VALUE = "-";
113120

114121

115122
private ImmutableMap<String, String> _clientIds;
@@ -184,14 +191,34 @@ public static String getTpcRequireChecksumVerification(HttpServletRequest reques
184191
return (String) request.getAttribute(TPC_REQUIRE_CHECKSUM_VERIFICATION_ATTRIBUTE);
185192
}
186193

194+
/**
195+
* Provide the remote source URI for pull-type third-party copies.
196+
*/
187197
public static URI getTpcSource(HttpServletRequest request) {
188198
return (URI) request.getAttribute(TPC_SOURCE_ATTRIBUTE);
189199
}
190200

201+
/**
202+
* Provide the remote destination URI for push-type third-party copies.
203+
*/
191204
public static URI getTpcDestination(HttpServletRequest request) {
192205
return (URI) request.getAttribute(TPC_DESTINATION_ATTRIBUTE);
193206
}
194207

208+
/**
209+
* Provide the SciTag header value supplied directly by the client.
210+
*/
211+
public static String getTpcSciTag(HttpServletRequest request) {
212+
return (String) request.getAttribute(TPC_SCITAG_ATTRIBUTE);
213+
}
214+
215+
/**
216+
* Provide the transfer-header-carried SciTag value supplied by the client.
217+
*/
218+
public static String getTpcTransferHeaderSciTag(HttpServletRequest request) {
219+
return (String) request.getAttribute(TPC_TRANSFER_HEADER_SCITAG_ATTRIBUTE);
220+
}
221+
195222
@Required
196223
public void setPathMapper(PathMapper mapper) {
197224
_pathMapper = mapper;
@@ -404,6 +431,9 @@ private void processThirdPartyCopy(Request request, Response response)
404431
throws BadRequestException, InterruptedException, ErrorResponseException {
405432
Direction direction = getDirection();
406433
URI remote = getRemoteLocation();
434+
HttpServletRequest servletRequest = ServletRequest.getRequest();
435+
436+
captureSciTagHeaders(servletRequest);
407437

408438
setRemoteUrlAttribute(direction, remote);
409439

@@ -434,12 +464,12 @@ private void processThirdPartyCopy(Request request, Response response)
434464

435465
var transferHeaders = buildTransferHeaders(request);
436466
var transferFlags = buildTransferFlags();
467+
String transferTag = transferTagForPool(servletRequest);
437468

438469
var transferResult = _remoteTransfers.acceptRequest(transferHeaders,
439470
getSubject(), getRestriction(), path, remote, credential,
440-
direction, transferFlags, overwriteAllowed, wantDigest);
471+
transferTag, direction, transferFlags, overwriteAllowed, wantDigest);
441472

442-
HttpServletRequest servletRequest = ServletRequest.getRequest();
443473
transferResult.addListener(() -> {
444474
try {
445475
var error = transferResult.get();
@@ -451,6 +481,36 @@ private void processThirdPartyCopy(Request request, Response response)
451481
}, MoreExecutors.directExecutor());
452482
}
453483

484+
private void captureSciTagHeaders(HttpServletRequest request) {
485+
request.setAttribute(TPC_SCITAG_ATTRIBUTE,
486+
sanitiseHeaderValue(request.getHeader(REQUEST_HEADER_SCITAG)));
487+
request.setAttribute(TPC_TRANSFER_HEADER_SCITAG_ATTRIBUTE,
488+
sanitiseHeaderValue(request.getHeader(REQUEST_HEADER_TRANSFER_HEADER_SCITAG)));
489+
}
490+
491+
private String sanitiseHeaderValue(String value) {
492+
if (value == null) {
493+
return MISSING_HEADER_VALUE;
494+
}
495+
496+
String trimmedValue = value.trim();
497+
return trimmedValue.isEmpty() ? MISSING_HEADER_VALUE : trimmedValue;
498+
}
499+
500+
private String transferTagForPool(HttpServletRequest request) {
501+
String sciTag = getTpcSciTag(request);
502+
if (sciTag != null && !MISSING_HEADER_VALUE.equals(sciTag)) {
503+
return sciTag;
504+
}
505+
506+
String transferHeaderSciTag = getTpcTransferHeaderSciTag(request);
507+
if (transferHeaderSciTag != null && !MISSING_HEADER_VALUE.equals(transferHeaderSciTag)) {
508+
return transferHeaderSciTag;
509+
}
510+
511+
return "";
512+
}
513+
454514
private void setRemoteUrlAttribute(Direction direction, URI remote) {
455515
HttpServletRequest request = ServletRequest.getRequest();
456516

‎modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -677,11 +677,12 @@ public void addTransfers(List<IoDoorEntry> transfers) {
677677
public ListenableFuture<Optional<String>> acceptRequest(
678678
ImmutableMap<String, String> transferHeaders,
679679
Subject subject, Restriction restriction, FsPath path, URI remote,
680-
Object credential, Direction direction, EnumSet<TransferFlag> flags,
680+
Object credential, String transferTag, Direction direction,
681+
EnumSet<TransferFlag> flags,
681682
boolean overwriteAllowed, Optional<String> wantDigest)
682683
throws ErrorResponseException, InterruptedException {
683684
RemoteTransfer transfer = new RemoteTransfer(subject, restriction,
684-
path, remote, credential, flags, transferHeaders, direction,
685+
path, remote, credential, transferTag, flags, transferHeaders, direction,
685686
overwriteAllowed, wantDigest);
686687

687688
return transfer.start();
@@ -763,6 +764,7 @@ private class RemoteTransfer {
763764
private final CredentialSource _source;
764765
private final EnumSet<TransferFlag> _flags;
765766
private final ImmutableMap<String, String> _transferHeaders;
767+
private final String _transferTag;
766768
private final Direction _direction;
767769
private final boolean _overwriteAllowed;
768770
private final Optional<String> _wantDigest;
@@ -787,6 +789,7 @@ private class RemoteTransfer {
787789

788790
public RemoteTransfer(Subject subject, Restriction restriction,
789791
FsPath path, URI destination, @Nullable Object credential,
792+
String transferTag,
790793
EnumSet<TransferFlag> flags, ImmutableMap<String, String> transferHeaders,
791794
Direction direction, boolean overwriteAllowed, Optional<String> wantDigest)
792795
throws ErrorResponseException {
@@ -818,6 +821,7 @@ public RemoteTransfer(Subject subject, Restriction restriction,
818821

819822
_flags = flags;
820823
_transferHeaders = transferHeaders;
824+
_transferTag = transferTag == null ? "" : transferTag.trim();
821825
_direction = direction;
822826
_overwriteAllowed = overwriteAllowed;
823827
_wantDigest = wantDigest;
@@ -1057,24 +1061,33 @@ private IpProtocolInfo buildProtocolInfo() throws ErrorResponseException {
10571061
null, desiredChecksum);
10581062

10591063
case HTTP:
1060-
return new RemoteHttpDataTransferProtocolInfo("RemoteHttpDataTransfer",
1064+
RemoteHttpDataTransferProtocolInfo httpInfo =
1065+
new RemoteHttpDataTransferProtocolInfo("RemoteHttpDataTransfer",
10611066
1, 1, address, _destination.toASCIIString(),
10621067
_flags.contains(TransferFlag.REQUIRE_VERIFICATION),
10631068
_transferHeaders, desiredChecksums);
1069+
httpInfo.setTransferTag(_transferTag);
1070+
return httpInfo;
10641071

10651072
case HTTPS:
10661073
if (_source == CredentialSource.OIDC) {
1067-
return new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer",
1074+
RemoteHttpsDataTransferProtocolInfo httpsInfo =
1075+
new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer",
10681076
1, 1, address, _destination.toASCIIString(),
10691077
_flags.contains(TransferFlag.REQUIRE_VERIFICATION),
10701078
_transferHeaders, desiredChecksums,
10711079
_oidCredential);
1080+
httpsInfo.setTransferTag(_transferTag);
1081+
return httpsInfo;
10721082
} else {
1073-
return new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer",
1083+
RemoteHttpsDataTransferProtocolInfo httpsInfo =
1084+
new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer",
10741085
1, 1, address, _destination.toASCIIString(),
10751086
_flags.contains(TransferFlag.REQUIRE_VERIFICATION),
10761087
_transferHeaders, _privateKey, _certificateChain,
10771088
desiredChecksums);
1089+
httpsInfo.setTransferTag(_transferTag);
1090+
return httpsInfo;
10781091
}
10791092
}
10801093

‎modules/dcache/src/main/java/org/dcache/pool/movers/TransferLifeCycle.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,8 @@ private boolean needMarker(ProtocolInfo protocolInfo) {
271271
case "xrootd":
272272
case "http":
273273
case "https":
274+
case "remotehttpdatatransfer":
275+
case "remotehttpsdatatransfer":
274276
return true;
275277
default:
276278
return false;

0 commit comments

Comments
 (0)