Merge branch 'develop'

CMakeLists.txt

@@ -45,7 +45,7 @@ set (OUTPUT_NAME_TRANSFER "gfal_transfer")
 
 set (VERSION_MAJOR  2)
 set (VERSION_MINOR  15)
-set (VERSION_PATCH  4)
+set (VERSION_PATCH  5)
 set (VERSION_STRING ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH})
 
 add_definitions(-DVERSION="${VERSION_STRING}")

packaging/debian/changelog

@@ -1,3 +1,8 @@
+gfal2 (2.15.5-1) unstable; urgency=low
+  * New version
+
+ -- DMC Devel <dmc-devel@cern.ch> Tue, 17 April 2018 16:00:00 +0100
+
 gfal2 (2.15.4-1) unstable; urgency=low
   * New version
 

packaging/rpm/gfal2.spec

@@ -4,7 +4,7 @@
 %bcond_with tests
 
 Name:               gfal2
-Version:            2.15.4
+Version:            2.15.5
 Release:            1%{?dist}
 Summary:            Grid file access library 2.0
 Group:              Applications/Internet

src/plugins/gridftp/gridftp_parsing.cpp

@@ -200,7 +200,9 @@ globus_result_t parse_mlst_line(char *line, struct stat *stat_info, char *filena
         }
 
         if (strcmp(startfact, "type") == 0) {
-            if (strcasecmp(factval, "dir") == 0) {
+            if ((strcasecmp(factval, "dir") == 0) 
+              || (strcasecmp(factval, "pdir") == 0)
+              || (strcasecmp(factval, "cdir") == 0))  {
                 type = GLOBUS_GASS_COPY_GLOB_ENTRY_DIR;
             }
             else if (strcasecmp(factval, "file") == 0) {
@@ -242,7 +244,7 @@ globus_result_t parse_mlst_line(char *line, struct stat *stat_info, char *filena
     stat_info->st_mtime = -1;
 
     if (mode_s) {
-        stat_info->st_mode = strtoul(mode_s, NULL, 0);
+        stat_info->st_mode = strtoul(mode_s, NULL, 8);
         if (type == GLOBUS_GASS_COPY_GLOB_ENTRY_DIR) {
             stat_info->st_mode |= S_IFDIR;
         }

src/plugins/srm/gfal_srm_getput.c

@@ -26,6 +26,7 @@
 #include "gfal_srm_request.h"
 #include "gfal_srm_internal_layer.h"
 #include "gfal_srm_endpoint.h"
+#include "gfal_srm_getput.h"
 
 
 // Make sure the TURL returned by the endpoint is one of the requested protocols

src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp

@@ -174,13 +174,13 @@ int gfal_xrootd_3rd_copy_bulk(plugin_handle plugin_data,
         GError*** file_errors)
 {
     GError* internalError = NULL;
-    char checksumType[64] = { 0 };
-    char checksumValue[512] = { 0 };
+    char _checksumType[64] = { 0 };
+    char _checksumValue[512] = { 0 };
     bool isThirdParty = false;
 
     gfalt_checksum_mode_t checksumMode = gfalt_get_checksum(params,
-        checksumType, sizeof(checksumType),
-        checksumValue, sizeof(checksumValue), NULL);
+        _checksumType, sizeof(_checksumType),
+        _checksumValue, sizeof(_checksumValue), NULL);
 
     XrdCl::CopyProcess copy_process;
 #if XrdMajorVNUM(XrdVNUMBER) == 4 ||  XrdMajorVNUM(XrdVNUMBER) == 100
@@ -235,11 +235,15 @@ int gfal_xrootd_3rd_copy_bulk(plugin_handle plugin_data,
 #endif
 
         if (checksumMode) {
-            checksumType[0] = '\0';
-            checksumValue[0] = '\0';
-            sscanf(checksums[i], "%63s:%511s", checksumType, checksumValue);
+            char checksumType[64] = { 0 };
+            char checksumValue[512] = { 0 };
+            char **chks = g_strsplit(checksums[i], ":", 2);
+            strncpy(checksumType, chks[0], sizeof(chks[0]));
+            strncpy(checksumValue,chks[1],  sizeof(chks[1]));
             checksumType[63] = checksumValue[511] = '\0';
-
+            g_strfreev(chks);
+	    gfal2_log(G_LOG_LEVEL_DEBUG, "Predefined Checksum Type: %s", checksumType);
+            gfal2_log(G_LOG_LEVEL_DEBUG, "Predefined Checksum Value: %s", checksumValue);
             if (!checksumType[0] || !checksumValue[0]) {
                 char* defaultChecksumType = gfal2_get_opt_string(context, XROOTD_CONFIG_GROUP, XROOTD_DEFAULT_CHECKSUM, &internalError);
                 if (internalError) {

src/plugins/xrootd/gfal_xrootd_plugin_bringonline.cpp

@@ -22,16 +22,15 @@
 #include <XrdCl/XrdClFileSystem.hh>
 #include <XrdSys/XrdSysPthread.hh>
 
-
 class PollResponseHandler: public XrdCl::ResponseHandler {
 private:
     XrdSysCondVar &condVar;
     GError **error;
     int &finishedCounter, &errCounter, &notAnsweredCounter;
-
+    const char* url;
 public:
-    PollResponseHandler(XrdSysCondVar &condVar, GError **error, int &finishedCounter, int &errCounter, int &notAnsweredCounter):
-        condVar(condVar), error(error), finishedCounter(finishedCounter), errCounter(errCounter), notAnsweredCounter(notAnsweredCounter) {
+    PollResponseHandler(const char* url, XrdSysCondVar &condVar, GError **error, int &finishedCounter, int &errCounter, int &notAnsweredCounter):
+        url(url),condVar(condVar), error(error), finishedCounter(finishedCounter), errCounter(errCounter), notAnsweredCounter(notAnsweredCounter) {
     }
 
     ~PollResponseHandler() {
@@ -40,33 +39,77 @@ class PollResponseHandler: public XrdCl::ResponseHandler {
     void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) {
         if (!status->IsOK()) {
             ++errCounter;
+            gfal2_log(G_LOG_LEVEL_DEBUG, "Error doing the query");
             gfal2_set_error(error, xrootd_domain,
-                xrootd_errno_to_posix_errno(status->errNo), __func__, "%s", status->GetErrorMessage().c_str());
+               xrootd_errno_to_posix_errno(status->errNo), __func__, "%s", status->GetErrorMessage().c_str());
         }
         delete status;
-
-        XrdCl::StatInfo *info = (XrdCl::StatInfo*)(response);
+
+        XrdCl::StatInfo *info = 0;
+        response->Get(info);
 
         condVar.Lock();
 
         --notAnsweredCounter;
         if (*error) {
             ++errCounter;
         }
-        else if (!info->TestFlags(XrdCl::StatInfo::Offline)) {
-            ++finishedCounter;
+        else if (!(info->TestFlags(XrdCl::StatInfo::Offline))) {
+           gfal2_log(G_LOG_LEVEL_DEBUG, "file online");  
+           ++finishedCounter;
         }
         else {
-            gfal2_set_error(error, xrootd_domain, EAGAIN, __func__, "Not online");
+            gfal2_log(G_LOG_LEVEL_DEBUG, "invoke the query for the error attribute for file: %s", url);
+            //invoke the query for the error attribute
+            XrdCl::Buffer arg;
+            XrdCl::Buffer *responsePtr;
+            XrdCl::URL endpoint(url);
+            XrdCl::FileSystem fs(endpoint);
+            //build the opaque
+            //
+            std::ostringstream sstr;
+            sstr << endpoint.GetPath() << "?mgm.pcmd=xattr&mgm.subcmd=get&mgm.xattrname=sys.retrieve.error";
+            arg.FromString(sstr.str());
+            gfal2_log(G_LOG_LEVEL_DEBUG, "attributes: %s", sstr.str().c_str());
+            XrdCl::Status st = fs.Query(XrdCl::QueryCode::Code::OpaqueFile , arg, responsePtr);
+
+	    std::unique_ptr<XrdCl::Buffer> res(responsePtr);
+
+            //TODO:what happens if the query fails?
+            if (!st.IsOK()) {
+                gfal2_log(G_LOG_LEVEL_DEBUG, "Error submitting query for extended attribute");
+                gfal2_set_error(error, xrootd_domain, EAGAIN, __func__, "%s","Not online");
+            } else {
+                //TODO: what we do if the response is empty
+	        if (!res->GetBuffer()) {
+                  gfal2_log(G_LOG_LEVEL_DEBUG, "Query for Extended Attribute is Empty");
+                } 
+                int retc;
+                char tag[1024];
+                char error_string[1024];
+                gfal2_log(G_LOG_LEVEL_DEBUG, "Response: %s", res->GetBuffer());
+                sscanf(res->GetBuffer(),
+                       "%s retc=%d value=%s",
+                       tag, &retc, error_string);
+	        //check the error string if it's not empty.
+	        if (retc || (error_string != "")) {
+                     gfal2_log(G_LOG_LEVEL_DEBUG, "Error reported: %s ", error_string);
+              	     gfal2_set_error(error, xrootd_domain, EIO, __func__, "%s",error_string);
+                     ++errCounter;
+                } else {
+                     gfal2_log(G_LOG_LEVEL_DEBUG, "No error reported");
+                     gfal2_set_error(error, xrootd_domain, EAGAIN, __func__, "%s","Not online");
+               }
+            }
         }
+
         if (notAnsweredCounter <= 0) {
             condVar.UnLock();
             condVar.Signal();
             condVar.Lock();
         }
-
         condVar.UnLock();
-        delete info;
+        delete response;
     }
 
     // std::vector expects an = operator, and the default one is no good
@@ -132,10 +175,11 @@ int gfal_xrootd_bring_online_poll_list(plugin_handle plugin_data,
 
     // Make sure all handlers are in place before calling async code
     for (int i = 0; i < nbfiles; ++i) {
-        handlers.emplace_back(condVar, &err[i], finishedCounter, errCounter, notAnsweredCounter);
+        handlers.emplace_back(urls[i], condVar, &err[i], finishedCounter, errCounter, notAnsweredCounter);
     }
     for (int i = 0; i < nbfiles; ++i) {
         XrdCl::URL file(prepare_url(context, urls[i]));
+        gfal2_log(G_LOG_LEVEL_DEBUG, "Doing file stat to check if the file is online: %s ", file.GetPath().c_str());
         XrdCl::Status st = fs.Stat(file.GetPath(), &handlers[i]);
         if (!st.IsOK()) {
             condVar.Lock();
@@ -205,3 +249,40 @@ int gfal_xrootd_release_file(plugin_handle plugin_data,
     }
     return ret;
 }
+
+int gfal_xrootd_abort_files(plugin_handle plugin_data,
+    int nbfiles, const char* const* urls, const char* token, GError** err)
+{
+    if (nbfiles <= 0) {
+        return 1;
+    }
+    gfal2_context_t context = (gfal2_context_t)plugin_data;
+
+    XrdCl::URL endpoint(urls[0]);
+    endpoint.SetPath(std::string());
+    XrdCl::FileSystem fs(endpoint);
+
+    std::vector<std::string> fileList;
+    for (int i = 0; i < nbfiles; ++i) {
+        XrdCl::URL file(prepare_url(context, urls[i]));
+        fileList.emplace_back(file.GetPath());
+    }
+
+    XrdCl::Buffer *reponsePtr;
+    //TODO : we use Fresh as a flag now, to change to Abort once it's implemented in xrootd
+    XrdCl::Status st = fs.Prepare(fileList, XrdCl::PrepareFlags::Flags::Fresh, 0, reponsePtr);
+    std::unique_ptr<XrdCl::Buffer> response(reponsePtr);
+
+    if (!st.IsOK()) {
+        GError *tmp_err = NULL;
+        gfal2_set_error(&tmp_err, xrootd_domain, xrootd_errno_to_posix_errno(st.errNo),
+            __func__, "%s", st.ToString().c_str());
+        for (int i = 0; i < nbfiles; ++i) {
+            err[i] = g_error_copy(tmp_err);
+        }
+        g_error_free(tmp_err);
+        return -1;
+    }
+    return 0;
+}
+

src/plugins/xrootd/gfal_xrootd_plugin_interface.h

@@ -114,6 +114,9 @@ int gfal_xrootd_bring_online_poll_list(plugin_handle plugin_data,
 int gfal_xrootd_release_file_list(plugin_handle plugin_data,
     int nbfiles, const char* const* urls, const char* token, GError** err);
 
+int gfal_xrootd_abort_files(plugin_handle plugin_data,
+    int nbfiles, const char* const* urls, const char* token, GError** err);
+
 const char* gfal_xrootd_getName();
 
 void set_xrootd_log_level();

src/plugins/xrootd/gfal_xrootd_plugin_main.cpp

@@ -87,6 +87,7 @@ gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError** err)
     xrootd_plugin.bring_online_poll_list = &gfal_xrootd_bring_online_poll_list;
     xrootd_plugin.release_file = &gfal_xrootd_release_file;
     xrootd_plugin.release_file_list = &gfal_xrootd_release_file_list;
+    xrootd_plugin.abort_files = &gfal_xrootd_abort_files;
 
     return xrootd_plugin;
 }