Actual source code: fretrieve.c

      1: /*
      2:       Code for opening and closing files.
      3: */
      4: #include <petscsys.h>
      5: #if defined(PETSC_HAVE_PWD_H)
      6:   #include <pwd.h>
      7: #endif
      8: #include <ctype.h>
      9: #include <sys/stat.h>
     10: #if defined(PETSC_HAVE_UNISTD_H)
     11:   #include <unistd.h>
     12: #endif
     13: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
     14:   #include <sys/utsname.h>
     15: #endif
     16: #include <fcntl.h>
     17: #include <time.h>
     18: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
     19:   #include <sys/systeminfo.h>
     20: #endif
     21: #include <petsc/private/petscimpl.h>

     23: /*
     24:    Private routine to delete tmp/shared storage

     26:    This is called by MPI, not by users, when communicator attributes are deleted

     28: */
     29: static PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state)
     30: {
     31:   PetscFunctionBegin;
     32:   PetscCallReturnMPI(PetscInfo(NULL, "Deleting tmp/shared data in an MPI_Comm %ld\n", (long)comm));
     33:   PetscCallReturnMPI(PetscFree(count_val));
     34:   PetscFunctionReturn(MPI_SUCCESS);
     35: }

     37: // "Unknown section 'Environmental Variables'"
     38: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
     39: /*@C
     40:   PetscGetTmp - Gets the name of the "tmp" directory, often this is `/tmp`

     42:   Collective

     44:   Input Parameters:
     45: + comm - MPI_Communicator that may share tmp
     46: - len  - length of string to hold name

     48:   Output Parameter:
     49: . dir - directory name

     51:   Environmental Variables:
     52: + `PETSC_SHARED_TMP`     - indicates the directory is known to be shared among the MPI processes
     53: . `PETSC_NOT_SHARED_TMP` - indicates the directory is known to be not shared among the MPI processes
     54: - `PETSC_TMP`            - name of the directory you wish to use as tmp

     56:   Level: developer

     58: .seealso: `PetscSharedTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
     59: @*/
     60: PetscErrorCode PetscGetTmp(MPI_Comm comm, char dir[], size_t len)
     61: {
     62:   PetscBool flg;

     64:   PetscFunctionBegin;
     65:   PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", dir, len, &flg));
     66:   if (!flg) PetscCall(PetscStrncpy(dir, "/tmp", len));
     67:   PetscFunctionReturn(PETSC_SUCCESS);
     68: }

     70: // "Unknown section 'Environmental Variables'"
     71: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
     72: /*@
     73:   PetscSharedTmp - Determines if all processors in a communicator share a
     74:   tmp directory or have different ones.

     76:   Collective

     78:   Input Parameter:
     79: . comm - MPI_Communicator that may share tmp

     81:   Output Parameter:
     82: . shared - `PETSC_TRUE` or `PETSC_FALSE`

     84:   Options Database Keys:
     85: + -shared_tmp     - indicates the directory is known to be shared among the MPI processes
     86: . -not_shared_tmp - indicates the directory is known to be not shared among the MPI processes
     87: - -tmp tmpdir     - name of the directory you wish to use as tmp

     89:   Environmental Variables:
     90: + `PETSC_SHARED_TMP`     - indicates the directory is known to be shared among the MPI processes
     91: . `PETSC_NOT_SHARED_TMP` - indicates the directory is known to be not shared among the MPI processes
     92: - `PETSC_TMP`            - name of the directory you wish to use as tmp

     94:   Level: developer

     96:   Notes:
     97:   Stores the status as a MPI attribute so it does not have
     98:   to be redetermined each time.

    100:   Assumes that all processors in a communicator either
    101:   1) have a common tmp or
    102:   2) each has a separate tmp
    103:   eventually we can write a fancier one that determines which processors
    104:   share a common tmp.

    106:   This will be very slow on runs with a large number of processors since
    107:   it requires O(p*p) file opens.

    109:   If the environmental variable `PETSC_TMP` is set it will use this directory
    110:   as the "tmp" directory.

    112: .seealso: `PetscGetTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
    113: @*/
    114: PetscErrorCode PetscSharedTmp(MPI_Comm comm, PetscBool *shared)
    115: {
    116:   PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
    117:   PetscBool   flg;
    118:   PetscMPIInt iflg;
    119:   FILE       *fd;
    120:   int         err;

    122:   PetscFunctionBegin;
    123:   PetscCallMPI(MPI_Comm_size(comm, &size));
    124:   if (size == 1) {
    125:     *shared = PETSC_TRUE;
    126:     PetscFunctionReturn(PETSC_SUCCESS);
    127:   }

    129:   PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_TMP", NULL, 0, &flg));
    130:   if (flg) {
    131:     *shared = PETSC_TRUE;
    132:     PetscFunctionReturn(PETSC_SUCCESS);
    133:   }

    135:   PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_TMP", NULL, 0, &flg));
    136:   if (flg) {
    137:     *shared = PETSC_FALSE;
    138:     PetscFunctionReturn(PETSC_SUCCESS);
    139:   }

    141:   if (Petsc_SharedTmp_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedTmp_keyval, NULL));

    143:   PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedTmp_keyval, (void **)&tagvalp, &iflg));
    144:   if (!iflg) {
    145:     char filename[PETSC_MAX_PATH_LEN], tmpname[PETSC_MAX_PATH_LEN];

    147:     /* This communicator does not yet have a shared tmp attribute */
    148:     PetscCall(PetscMalloc1(1, &tagvalp));
    149:     PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedTmp_keyval, tagvalp));

    151:     PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", tmpname, 238, &flg));
    152:     if (!flg) {
    153:       PetscCall(PetscStrncpy(filename, "/tmp", sizeof(filename)));
    154:     } else {
    155:       PetscCall(PetscStrncpy(filename, tmpname, sizeof(filename)));
    156:     }

    158:     PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
    159:     PetscCallMPI(MPI_Comm_rank(comm, &rank));

    161:     /* each processor creates a /tmp file and all the later ones check */
    162:     /* this makes sure no subset of processors is shared */
    163:     *shared = PETSC_FALSE;
    164:     for (i = 0; i < size - 1; i++) {
    165:       if (rank == i) {
    166:         fd = fopen(filename, "w");
    167:         PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
    168:         err = fclose(fd);
    169:         PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
    170:       }
    171:       PetscCallMPI(MPI_Barrier(comm));
    172:       if (rank >= i) {
    173:         fd = fopen(filename, "r");
    174:         if (fd) cnt = 1;
    175:         else cnt = 0;
    176:         if (fd) {
    177:           err = fclose(fd);
    178:           PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
    179:         }
    180:       } else cnt = 0;

    182:       PetscCallMPI(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
    183:       if (rank == i) unlink(filename);

    185:       if (sum == size) {
    186:         *shared = PETSC_TRUE;
    187:         break;
    188:       } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share /tmp ");
    189:     }
    190:     *tagvalp = (int)*shared;
    191:     PetscCall(PetscInfo(NULL, "processors %s %s\n", *shared ? "share" : "do NOT share", flg ? tmpname : "/tmp"));
    192:   } else *shared = (PetscBool)*tagvalp;
    193:   PetscFunctionReturn(PETSC_SUCCESS);
    194: }

    196: // "Unknown section 'Environmental Variables'"
    197: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
    198: /*@
    199:   PetscSharedWorkingDirectory - Determines if all processors in a communicator share a working directory or have different ones.

    201:   Collective

    203:   Input Parameter:
    204: . comm - MPI_Communicator that may share working directory

    206:   Output Parameter:
    207: . shared - `PETSC_TRUE` or `PETSC_FALSE`

    209:   Options Database Keys:
    210: + -shared_working_directory     - indicates the directory is known to be shared among the MPI processes
    211: - -not_shared_working_directory - indicates the directory is known to be not shared among the MPI processes

    213:   Environmental Variables:
    214: + `PETSC_SHARED_WORKING_DIRECTORY`     - indicates the directory is known to be shared among the MPI processes
    215: - `PETSC_NOT_SHARED_WORKING_DIRECTORY` - indicates the directory is known to be not shared among the MPI processes

    217:   Level: developer

    219:   Notes:
    220:   Stores the status as a MPI attribute so it does not have to be redetermined each time.

    222:   Assumes that all processors in a communicator either
    223: .vb
    224:    1) have a common working directory or
    225:    2) each has a separate working directory
    226: .ve
    227:   eventually we can write a fancier one that determines which processors share a common working directory.

    229:   This will be very slow on runs with a large number of processors since it requires O(p*p) file opens.

    231: .seealso: `PetscGetTmp()`, `PetscSharedTmp()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
    232: @*/
    233: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm, PetscBool *shared)
    234: {
    235:   PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
    236:   PetscBool   flg;
    237:   PetscMPIInt iflg;
    238:   FILE       *fd;
    239:   int         err;

    241:   PetscFunctionBegin;
    242:   PetscCallMPI(MPI_Comm_size(comm, &size));
    243:   if (size == 1) {
    244:     *shared = PETSC_TRUE;
    245:     PetscFunctionReturn(PETSC_SUCCESS);
    246:   }

    248:   PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
    249:   if (flg) {
    250:     *shared = PETSC_TRUE;
    251:     PetscFunctionReturn(PETSC_SUCCESS);
    252:   }

    254:   PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
    255:   if (flg) {
    256:     *shared = PETSC_FALSE;
    257:     PetscFunctionReturn(PETSC_SUCCESS);
    258:   }

    260:   if (Petsc_SharedWD_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedWD_keyval, NULL));

    262:   PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedWD_keyval, (void **)&tagvalp, &iflg));
    263:   if (!iflg) {
    264:     char filename[PETSC_MAX_PATH_LEN];

    266:     /* This communicator does not yet have a shared  attribute */
    267:     PetscCall(PetscMalloc1(1, &tagvalp));
    268:     PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedWD_keyval, tagvalp));

    270:     PetscCall(PetscGetWorkingDirectory(filename, sizeof(filename) - 16));
    271:     PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
    272:     PetscCallMPI(MPI_Comm_rank(comm, &rank));

    274:     /* each processor creates a  file and all the later ones check */
    275:     /* this makes sure no subset of processors is shared */
    276:     *shared = PETSC_FALSE;
    277:     for (i = 0; i < size - 1; i++) {
    278:       if (rank == i) {
    279:         fd = fopen(filename, "w");
    280:         PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
    281:         err = fclose(fd);
    282:         PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
    283:       }
    284:       PetscCallMPI(MPI_Barrier(comm));
    285:       if (rank >= i) {
    286:         fd = fopen(filename, "r");
    287:         if (fd) cnt = 1;
    288:         else cnt = 0;
    289:         if (fd) {
    290:           err = fclose(fd);
    291:           PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
    292:         }
    293:       } else cnt = 0;

    295:       PetscCallMPI(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
    296:       if (rank == i) unlink(filename);

    298:       if (sum == size) {
    299:         *shared = PETSC_TRUE;
    300:         break;
    301:       } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share working directory");
    302:     }
    303:     *tagvalp = (int)*shared;
    304:   } else *shared = (PetscBool)*tagvalp;
    305:   PetscCall(PetscInfo(NULL, "processors %s working directory\n", (*shared) ? "shared" : "do NOT share"));
    306:   PetscFunctionReturn(PETSC_SUCCESS);
    307: }

    309: /*@C
    310:   PetscFileRetrieve - Obtains a file from a URL or a compressed file
    311:   and copies into local disk space as uncompressed.

    313:   Collective

    315:   Input Parameters:
    316: + comm - processors accessing the file
    317: . url  - name of file, including entire URL (with or without .gz)
    318: - llen - length of `localname`

    320:   Output Parameters:
    321: + localname - name of local copy of file - valid on only process zero
    322: - found     - if found or retrieved the file - valid on all processes

    324:   Level: developer

    326:   Note:
    327:   if the file already exists locally this function just returns without downloading it.

    329: .seealso: `PetscDLLibraryRetrieve()`
    330: @*/
    331: PetscErrorCode PetscFileRetrieve(MPI_Comm comm, const char url[], char localname[], size_t llen, PetscBool *found)
    332: {
    333:   char        buffer[PETSC_MAX_PATH_LEN], *par = NULL, *tlocalname = NULL, name[PETSC_MAX_PATH_LEN];
    334:   FILE       *fp;
    335:   PetscMPIInt rank;
    336:   size_t      len = 0;
    337:   PetscBool   flg1, flg2, flg3, flg4, download, compressed = PETSC_FALSE;

    339:   PetscFunctionBegin;
    340:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
    341:   if (rank == 0) {
    342:     *found = PETSC_FALSE;

    344:     PetscCall(PetscStrstr(url, ".gz", &par));
    345:     if (par) {
    346:       PetscCall(PetscStrlen(par, &len));
    347:       if (len == 3) compressed = PETSC_TRUE;
    348:     }

    350:     PetscCall(PetscStrncmp(url, "ftp://", 6, &flg1));
    351:     PetscCall(PetscStrncmp(url, "http://", 7, &flg2));
    352:     PetscCall(PetscStrncmp(url, "file://", 7, &flg3));
    353:     PetscCall(PetscStrncmp(url, "https://", 8, &flg4));
    354:     download = (PetscBool)(flg1 || flg2 || flg3 || flg4);

    356:     if (!download && !compressed) {
    357:       PetscCall(PetscStrncpy(localname, url, llen));
    358:       PetscCall(PetscTestFile(url, 'r', found));
    359:       if (*found) {
    360:         PetscCall(PetscInfo(NULL, "Found file %s\n", url));
    361:       } else {
    362:         PetscCall(PetscInfo(NULL, "Did not find file %s\n", url));
    363:       }
    364:       goto done;
    365:     }

    367:     /* look for uncompressed file in requested directory */
    368:     if (compressed) {
    369:       PetscCall(PetscStrncpy(localname, url, llen));
    370:       PetscCall(PetscStrstr(localname, ".gz", &par));
    371:       *par = 0; /* remove .gz extension */
    372:       PetscCall(PetscTestFile(localname, 'r', found));
    373:       if (*found) goto done;
    374:     }

    376:     /* look for file in current directory */
    377:     PetscCall(PetscStrrchr(url, '/', &tlocalname));
    378:     PetscCall(PetscStrncpy(localname, tlocalname, llen));
    379:     if (compressed) {
    380:       PetscCall(PetscStrstr(localname, ".gz", &par));
    381:       *par = 0; /* remove .gz extension */
    382:     }
    383:     PetscCall(PetscTestFile(localname, 'r', found));
    384:     if (*found) goto done;

    386:     if (download) {
    387:       /* local file is not already here so use curl to get it */
    388:       PetscCall(PetscStrncpy(localname, tlocalname, llen));
    389:       PetscCall(PetscStrncpy(buffer, "curl --fail --silent --show-error ", sizeof(buffer)));
    390:       PetscCall(PetscStrlcat(buffer, url, sizeof(buffer)));
    391:       PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
    392:       PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
    393: #if defined(PETSC_HAVE_POPEN)
    394:       PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
    395:       PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
    396: #else
    397:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
    398: #endif
    399:       PetscCall(PetscTestFile(localname, 'r', found));
    400:       if (*found) {
    401:         FILE *fd;
    402:         char  buf[1024], *str, *substring;

    404:         /* check if the file didn't exist so it downloaded an HTML message instead */
    405:         fd = fopen(localname, "r");
    406:         PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscTestFile() indicates %s exists but fopen() cannot open it", localname);
    407:         str = fgets(buf, sizeof(buf) - 1, fd);
    408:         while (str) {
    409:           PetscCall(PetscStrstr(buf, "<!DOCTYPE html>", &substring));
    410:           PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
    411:           PetscCall(PetscStrstr(buf, "Not Found", &substring));
    412:           PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
    413:           str = fgets(buf, sizeof(buf) - 1, fd);
    414:         }
    415:         fclose(fd);
    416:       }
    417:     } else if (compressed) {
    418:       PetscCall(PetscTestFile(url, 'r', found));
    419:       if (!*found) goto done;
    420:       PetscCall(PetscStrncpy(localname, url, llen));
    421:     }
    422:     if (compressed) {
    423:       PetscCall(PetscStrrchr(localname, '/', &tlocalname));
    424:       PetscCall(PetscStrncpy(name, tlocalname, PETSC_MAX_PATH_LEN));
    425:       PetscCall(PetscStrstr(name, ".gz", &par));
    426:       *par = 0; /* remove .gz extension */
    427:       /* uncompress file */
    428:       PetscCall(PetscStrncpy(buffer, "gzip -c -d ", sizeof(buffer)));
    429:       PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
    430:       PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
    431:       PetscCall(PetscStrlcat(buffer, name, sizeof(buffer)));
    432: #if defined(PETSC_HAVE_POPEN)
    433:       PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
    434:       PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
    435: #else
    436:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
    437: #endif
    438:       PetscCall(PetscStrncpy(localname, name, llen));
    439:       PetscCall(PetscTestFile(localname, 'r', found));
    440:     }
    441:   }
    442: done:
    443:   PetscCallMPI(MPI_Bcast(found, 1, MPI_C_BOOL, 0, comm));
    444:   PetscCallMPI(MPI_Bcast(localname, (PetscMPIInt)llen, MPI_CHAR, 0, comm));
    445:   PetscFunctionReturn(PETSC_SUCCESS);
    446: }