Actual source code: fretrieve.c
2: /*
3: Code for opening and closing files.
4: */
5: #include <petscsys.h>
6: #if defined(PETSC_HAVE_PWD_H)
7: #include <pwd.h>
8: #endif
9: #include <ctype.h>
10: #include <sys/stat.h>
11: #if defined(PETSC_HAVE_UNISTD_H)
12: #include <unistd.h>
13: #endif
14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
15: #include <sys/utsname.h>
16: #endif
17: #include <fcntl.h>
18: #include <time.h>
19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
20: #include <sys/systeminfo.h>
21: #endif
23: /*
24: Private routine to delete tmp/shared storage
26: This is called by MPI, not by users.
28: Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
30: */
31: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
32: {
36: PetscInfo1(NULL,"Deleting tmp/shared data in an MPI_Comm %ld\n",(long)comm);
37: PetscFree(count_val);
38: PetscFunctionReturn(MPI_SUCCESS);
39: }
41: /*@C
42: PetscGetTmp - Gets the name of the tmp directory
44: Collective
46: Input Parameters:
47: + comm - MPI_Communicator that may share /tmp
48: - len - length of string to hold name
50: Output Parameters:
51: . dir - directory name
53: Options Database Keys:
54: + -shared_tmp - indicates the directory is shared among the MPI ranks
55: . -not_shared_tmp - indicates the directory is not shared among the MPI ranks
56: - -tmp tmpdir - name of the directory you wish to use as /tmp
58: Environmental Variables:
59: + PETSC_SHARED_TMP - indicates the directory is shared among the MPI ranks
60: . PETSC_NOT_SHARED_TMP - indicates the directory is not shared among the MPI ranks
61: - PETSC_TMP - name of the directory you wish to use as /tmp
63: Level: developer
66: @*/
67: PetscErrorCode PetscGetTmp(MPI_Comm comm,char dir[],size_t len)
68: {
70: PetscBool flg;
73: PetscOptionsGetenv(comm,"PETSC_TMP",dir,len,&flg);
74: if (!flg) {
75: PetscStrncpy(dir,"/tmp",len);
76: }
77: return(0);
78: }
80: /*@C
81: PetscSharedTmp - Determines if all processors in a communicator share a
82: /tmp or have different ones.
84: Collective
86: Input Parameters:
87: . comm - MPI_Communicator that may share /tmp
89: Output Parameters:
90: . shared - PETSC_TRUE or PETSC_FALSE
92: Options Database Keys:
93: + -shared_tmp - indicates the directory is shared among the MPI ranks
94: . -not_shared_tmp - indicates the directory is not shared among the MPI ranks
95: - -tmp tmpdir - name of the directory you wish to use as /tmp
97: Environmental Variables:
98: + PETSC_SHARED_TMP - indicates the directory is shared among the MPI ranks
99: . PETSC_NOT_SHARED_TMP - indicates the directory is not shared among the MPI ranks
100: - PETSC_TMP - name of the directory you wish to use as /tmp
102: Level: developer
104: Notes:
105: Stores the status as a MPI attribute so it does not have
106: to be redetermined each time.
108: Assumes that all processors in a communicator either
109: 1) have a common /tmp or
110: 2) each has a separate /tmp
111: eventually we can write a fancier one that determines which processors
112: share a common /tmp.
114: This will be very slow on runs with a large number of processors since
115: it requires O(p*p) file opens.
117: If the environmental variable PETSC_TMP is set it will use this directory
118: as the "/tmp" directory.
120: @*/
121: PetscErrorCode PetscSharedTmp(MPI_Comm comm,PetscBool *shared)
122: {
123: PetscErrorCode ierr;
124: PetscMPIInt size,rank,*tagvalp,sum,cnt,i;
125: PetscBool flg,iflg;
126: FILE *fd;
127: static PetscMPIInt Petsc_Tmp_keyval = MPI_KEYVAL_INVALID;
128: int err;
131: MPI_Comm_size(comm,&size);
132: if (size == 1) {
133: *shared = PETSC_TRUE;
134: return(0);
135: }
137: PetscOptionsGetenv(comm,"PETSC_SHARED_TMP",NULL,0,&flg);
138: if (flg) {
139: *shared = PETSC_TRUE;
140: return(0);
141: }
143: PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_TMP",NULL,0,&flg);
144: if (flg) {
145: *shared = PETSC_FALSE;
146: return(0);
147: }
149: if (Petsc_Tmp_keyval == MPI_KEYVAL_INVALID) {
150: MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_Tmp_keyval,NULL);
151: }
153: MPI_Comm_get_attr(comm,Petsc_Tmp_keyval,(void**)&tagvalp,(int*)&iflg);
154: if (!iflg) {
155: char filename[PETSC_MAX_PATH_LEN],tmpname[PETSC_MAX_PATH_LEN];
157: /* This communicator does not yet have a shared tmp attribute */
158: PetscMalloc1(1,&tagvalp);
159: MPI_Comm_set_attr(comm,Petsc_Tmp_keyval,tagvalp);
161: PetscOptionsGetenv(comm,"PETSC_TMP",tmpname,238,&iflg);
162: if (!iflg) {
163: PetscStrcpy(filename,"/tmp");
164: } else {
165: PetscStrcpy(filename,tmpname);
166: }
168: PetscStrcat(filename,"/petsctestshared");
169: MPI_Comm_rank(comm,&rank);
171: /* each processor creates a /tmp file and all the later ones check */
172: /* this makes sure no subset of processors is shared */
173: *shared = PETSC_FALSE;
174: for (i=0; i<size-1; i++) {
175: if (rank == i) {
176: fd = fopen(filename,"w");
177: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
178: err = fclose(fd);
179: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
180: }
181: MPI_Barrier(comm);
182: if (rank >= i) {
183: fd = fopen(filename,"r");
184: if (fd) cnt = 1;
185: else cnt = 0;
186: if (fd) {
187: err = fclose(fd);
188: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
189: }
190: } else cnt = 0;
192: MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
193: if (rank == i) unlink(filename);
195: if (sum == size) {
196: *shared = PETSC_TRUE;
197: break;
198: } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share /tmp ");
199: }
200: *tagvalp = (int)*shared;
201: PetscInfo2(NULL,"processors %s %s\n",(*shared) ? "share":"do NOT share",(iflg ? tmpname:"/tmp"));
202: } else *shared = (PetscBool) *tagvalp;
203: return(0);
204: }
206: /*@C
207: PetscSharedWorkingDirectory - Determines if all processors in a communicator share a
208: working directory or have different ones.
210: Collective
212: Input Parameters:
213: . comm - MPI_Communicator that may share working directory
215: Output Parameters:
216: . shared - PETSC_TRUE or PETSC_FALSE
218: Options Database Keys:
219: + -shared_working_directory - indicates the directory is shared among the MPI ranks
220: - -not_shared_working_directory - indicates the directory is shared among the MPI ranks
222: Environmental Variables:
223: + PETSC_SHARED_WORKING_DIRECTORY - indicates the directory is shared among the MPI ranks
224: - PETSC_NOT_SHARED_WORKING_DIRECTORY - indicates the directory is shared among the MPI ranks
226: Level: developer
228: Notes:
229: Stores the status as a MPI attribute so it does not have
230: to be redetermined each time.
232: Assumes that all processors in a communicator either
233: 1) have a common working directory or
234: 2) each has a separate working directory
235: eventually we can write a fancier one that determines which processors
236: share a common working directory.
238: This will be very slow on runs with a large number of processors since
239: it requires O(p*p) file opens.
241: @*/
242: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm,PetscBool *shared)
243: {
244: PetscErrorCode ierr;
245: PetscMPIInt size,rank,*tagvalp,sum,cnt,i;
246: PetscBool flg,iflg;
247: FILE *fd;
248: static PetscMPIInt Petsc_WD_keyval = MPI_KEYVAL_INVALID;
249: int err;
252: MPI_Comm_size(comm,&size);
253: if (size == 1) {
254: *shared = PETSC_TRUE;
255: return(0);
256: }
258: PetscOptionsGetenv(comm,"PETSC_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
259: if (flg) {
260: *shared = PETSC_TRUE;
261: return(0);
262: }
264: PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
265: if (flg) {
266: *shared = PETSC_FALSE;
267: return(0);
268: }
270: if (Petsc_WD_keyval == MPI_KEYVAL_INVALID) {
271: MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_WD_keyval,NULL);
272: }
274: MPI_Comm_get_attr(comm,Petsc_WD_keyval,(void**)&tagvalp,(int*)&iflg);
275: if (!iflg) {
276: char filename[PETSC_MAX_PATH_LEN];
278: /* This communicator does not yet have a shared attribute */
279: PetscMalloc1(1,&tagvalp);
280: MPI_Comm_set_attr(comm,Petsc_WD_keyval,tagvalp);
282: PetscGetWorkingDirectory(filename,240);
283: PetscStrcat(filename,"/petsctestshared");
284: MPI_Comm_rank(comm,&rank);
286: /* each processor creates a file and all the later ones check */
287: /* this makes sure no subset of processors is shared */
288: *shared = PETSC_FALSE;
289: for (i=0; i<size-1; i++) {
290: if (rank == i) {
291: fd = fopen(filename,"w");
292: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
293: err = fclose(fd);
294: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
295: }
296: MPI_Barrier(comm);
297: if (rank >= i) {
298: fd = fopen(filename,"r");
299: if (fd) cnt = 1;
300: else cnt = 0;
301: if (fd) {
302: err = fclose(fd);
303: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
304: }
305: } else cnt = 0;
307: MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
308: if (rank == i) unlink(filename);
310: if (sum == size) {
311: *shared = PETSC_TRUE;
312: break;
313: } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share working directory");
314: }
315: *tagvalp = (int)*shared;
316: } else *shared = (PetscBool) *tagvalp;
317: PetscInfo1(NULL,"processors %s working directory\n",(*shared) ? "shared" : "do NOT share");
318: return(0);
319: }
322: /*@C
323: PetscFileRetrieve - Obtains a file from a URL or compressed
324: and copies into local disk space as uncompressed.
326: Collective
328: Input Parameter:
329: + comm - processors accessing the file
330: . url - name of file, including entire URL (with or without .gz)
331: - llen - length of localname
333: Output Parameter:
334: + localname - name of local copy of file - valid on only process zero
335: - found - if found or retrieved the file - valid on all processes
337: Notes:
338: if the file already exists local this function just returns without downloading it.
340: Level: intermediate
341: @*/
342: PetscErrorCode PetscFileRetrieve(MPI_Comm comm,const char url[],char localname[],size_t llen,PetscBool *found)
343: {
344: char buffer[PETSC_MAX_PATH_LEN],*par,*tlocalname,name[PETSC_MAX_PATH_LEN];
345: FILE *fp;
347: PetscMPIInt rank;
348: size_t len = 0;
349: PetscBool flg1,flg2,flg3,flg4,download,compressed = PETSC_FALSE;
352: MPI_Comm_rank(comm,&rank);
353: if (!rank) {
354: *found = PETSC_FALSE;
356: PetscStrstr(url,".gz",&par);
357: if (par) {
358: PetscStrlen(par,&len);
359: if (len == 3) compressed = PETSC_TRUE;
360: }
362: PetscStrncmp(url,"ftp://",6,&flg1);
363: PetscStrncmp(url,"http://",7,&flg2);
364: PetscStrncmp(url,"file://",7,&flg3);
365: PetscStrncmp(url,"https://",8,&flg4);
366: download = (PetscBool) (flg1 || flg2 || flg3 || flg4);
368: if (!download && !compressed) {
369: PetscStrncpy(localname,url,llen);
370: PetscTestFile(url,'r',found);
371: if (*found) {
372: PetscInfo1(NULL,"Found file %s\n",url);
373: } else {
374: PetscInfo1(NULL,"Did not find file %s\n",url);
375: }
376: goto done;
377: }
379: /* look for uncompressed file in requested directory */
380: if (compressed) {
381: PetscStrncpy(localname,url,llen);
382: PetscStrstr(localname,".gz",&par);
383: *par = 0; /* remove .gz extension */
384: PetscTestFile(localname,'r',found);
385: if (*found) goto done;
386: }
388: /* look for file in current directory */
389: PetscStrrchr(url,'/',&tlocalname);
390: PetscStrncpy(localname,tlocalname,llen);
391: if (compressed) {
392: PetscStrstr(localname,".gz",&par);
393: *par = 0; /* remove .gz extension */
394: }
395: PetscTestFile(localname,'r',found);
396: if (*found) goto done;
398: if (download) {
399: /* local file is not already here so use curl to get it */
400: PetscStrncpy(localname,tlocalname,llen);
401: PetscStrcpy(buffer,"curl --fail --silent --show-error ");
402: PetscStrcat(buffer,url);
403: PetscStrcat(buffer," > ");
404: PetscStrcat(buffer,localname);
405: #if defined(PETSC_HAVE_POPEN)
406: PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
407: PetscPClose(PETSC_COMM_SELF,fp);
408: #else
409: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
410: #endif
411: PetscTestFile(localname,'r',found);
412: if (*found) {
413: FILE *fd;
414: char buf[1024],*str,*substring;
416: /* check if the file didn't exist so it downloaded an HTML message instead */
417: fd = fopen(localname,"r");
418: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"PetscTestFile() indicates %s exists but fopen() cannot open it",localname);
419: str = fgets(buf,sizeof(buf)-1,fd);
420: while (str) {
421: PetscStrstr(buf,"<!DOCTYPE html>",&substring);
422: if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
423: PetscStrstr(buf,"Not Found",&substring);
424: if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
425: str = fgets(buf,sizeof(buf)-1,fd);
426: }
427: fclose(fd);
428: }
429: } else if (compressed) {
430: PetscTestFile(url,'r',found);
431: if (!*found) goto done;
432: PetscStrncpy(localname,url,llen);
433: }
434: if (compressed) {
435: PetscStrrchr(localname,'/',&tlocalname);
436: PetscStrncpy(name,tlocalname,PETSC_MAX_PATH_LEN);
437: PetscStrstr(name,".gz",&par);
438: *par = 0; /* remove .gz extension */
439: /* uncompress file */
440: PetscStrcpy(buffer,"gzip -c -d ");
441: PetscStrcat(buffer,localname);
442: PetscStrcat(buffer," > ");
443: PetscStrcat(buffer,name);
444: #if defined(PETSC_HAVE_POPEN)
445: PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
446: PetscPClose(PETSC_COMM_SELF,fp);
447: #else
448: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
449: #endif
450: PetscStrncpy(localname,name,llen);
451: PetscTestFile(localname,'r',found);
452: }
453: }
454: done:
455: MPI_Bcast(found,1,MPIU_BOOL,0,comm);
456: MPI_Bcast(localname, llen, MPI_CHAR, 0, comm);
457: return(0);
458: }