Coverage for model_workflow/utils/httpsf.py: 0%

1#!/usr/bin/env python 

2 

3import os 

4import sys 

5import requests 

6import stat # For file mode constants 

7from fuse import FUSE, Operations # FUSE bindings with fusepy 

8 

9 

10# Set the functions which are set by default when the file handler is passed to the fuse class 

11# These functions are better not overwritten in the file handler 

12# These functions were found experimentally but there may be more 

13default_fuse_functions = ['init', 'destroy', 'getattr', 'access'] 

14 

15# Set the functions which the fuse standards may expect to find in any compliant class and are not yet implemented 

16# These functions are set as whistleblowers 

17# These functions were found in the internet: 

18# https://www.cs.hmc.edu/~geoff/classes/hmc.cs135.201109/homework/fuse/fuse_doc.html 

19# https://www.cs.hmc.edu/~geoff/classes/hmc.cs135.201109/homework/fuse/fuse_doc.html#gotchas 

20# https://libfuse.github.io/doxygen/structfuse__operations.html 

21not_implememnted_fuse_functions = ['fgetattr', 'readlink', 'opendir', 'readdir', 'mknod', 'mkdir', 'unlink', 

22 'rmdir', 'symlink', 'rename', 'link', 'chmod', 'chown', 'truncate', 'ftruncate', 'utimens', 'open', 'read', 

23 'write', 'statfs', 'release', 'releasedir', 'fsync', 'fsyncdir', 'flush', 'lock', 'bmap', 'setxattr', 

24 'getxattr', 'listxattr', 'removexattr', 'ioctl', 'poll', 'create', 'openat', 'write_buf', 'read_buf', 

25 'flock', 'fallocate', 'copy_file_range', 'lseek'] 

26 

27# FUSE implementation for a single file 

28class FileHandler(Operations): 

29 def __init__ (self, url : str): 

30 print("Initializing API Virtual File System...") 

31 self.url = url 

32 self._response = None 

33 # Set all missing fuse functions as whistleblowers 

34 # Most commands will return a clueless 'function not implemented' error when trying to access any of these functions 

35 # This prevents the error and intead tells the user which function is being called and not yet implemented 

36 # This double function is to avoid overwritting the same function every time 

37 def create_whistleblower (name : str): 

38 def whistleblower (*args): 

39 raise SystemExit('FUSE function not implemented: ' + name) 

40 return whistleblower 

41 for function_name in not_implememnted_fuse_functions: 

42 if hasattr(self, function_name): 

43 continue 

44 setattr(self, function_name, create_whistleblower(function_name)) 

45 

46 def __call__ (self, function_name : str, *args, **kwargs): 

47 function = getattr(self, function_name) 

48 return function(*args) 

49 

50 def _get_file_size(self): 

51 response = requests.head(self.url, allow_redirects=True) 

52 response.raise_for_status() # Raise an exception for HTTP errors 

53 # Check if Content-Length header exists 

54 size = int(response.headers.get('Content-Length', 0)) 

55 return size 

56 

57 def _fetch_file_content(self): 

58 """Fetch content from remote API.""" 

59 self._response = requests.get(self.url) 

60 self._response.raise_for_status() 

61 

62 # Attribute keys 

63 # Got from https://github.com/skorokithakis/python-fuse-sample/blob/master/passthrough.py 

64 stat_keys = 'st_atime', 'st_ctime', 'st_mtime', 'st_gid', 'st_uid', 'st_mode', 'st_nlink', 'st_size' 

65 

66 # This must be implemented, although there is a default function 

67 # FUSE relies in this function a lot and not implementing it properly leads to silent errors 

68 def getattr (self, path, fh=None): 

69 # Get the stat of current directory to get time and id values 

70 dot_st = os.lstat('.') 

71 # Example regular file: 

72 # (st_mode=33188, st_ino=13765084, st_dev=66306, st_nlink=1, st_uid=16556, st_gid=500, st_size=9410, 

73 # st_atime=1696945741, st_mtime=1696945600, st_ctime=1696945600) 

74 # Example fifo 

75 # (st_mode=4516, st_ino=13765935, st_dev=66306, st_nlink=1, st_uid=16556, st_gid=500, st_size=0, 

76 # st_atime=1697017261, st_mtime=1697017261, st_ctime=1697017261) 

77 st = dict((key, getattr(dot_st, key)) for key in self.stat_keys) 

78 # Set a size bigger than the expected response, no matter how much bigger 

79 # DANI: Intenté muchas formas de que la size fuese nula o infinita para que no haya limite, pero no pude 

80 # DANI: Prové a cambiar el st_mode para convertir el punto de montaje en ostras cosas, pero solo puede ser file 

81 #stat['st_size'] = sys.maxsize 

82 st['st_size'] = self._get_file_size() 

83 # Set a custom mode for FUSE to treat it as a file with infinite size 

84 # To see the actual meaningful value of st_mode you have to do 'oct(st_mode)' 

85 # e.g. regular file: 33188 -> 0o100644, fifo: 4516 -> 0o10644 

86 # Last three numbers stand for the permissions 

87 # The starting numbers stand for the file type 

88 # UNIX file type docs 

89 # https://man7.org/linux/man-pages/man7/inode.7.html (section 'The file type and mode') 

90 # Set the mode for a FIFO 

91 st['st_mode'] = 0o644 | stat.S_IFIFO 

92 return st 

93 

94 # File methods 

95 # ============ 

96 

97 # Open the URL request 

98 def open (self, path : str, flags) -> int: 

99 # Lazy load content if not already fetched 

100 if self._response is None: 

101 self._fetch_file_content() 

102 # Return a file descriptor (dummy in this case) 

103 return 0 

104 

105 # Read length is set automatically by FUSE 

106 # This length is calculated from the getattr 'st_size' value 

107 # However the length is not the exact value of st_size, but the first multiple of 4096 which exceeds its value 

108 # Also length has a maximum value: 131072, experimentally observed  

109 # When the maximum value is not enought to cover the length the read function is called several times 

110 # Each time with different offset value 

111 def read (self, path : str, length : int, offset : int, fh : int) -> str: 

112 if self._response is None: 

113 self._fetch_file_content() 

114 return self._response.content[offset:offset + length] 

115 

116 # Close the URL request 

117 def destroy (self, path : str): 

118 if self._response: 

119 self._response.close() 

120 

121 # The following functions are not implemented 

122 # However they must be defined or FUSE complains when trying to read the file 

123 flush = None 

124 release = None 

125 lock = None 

126 # Note the FUSE will complain for several functions to be not implemented when trying to write in the file 

127 # User should never try to write this file so these cases are not handled 

128 

129# Mount a fake file which returns an url content when read 

130def mount (url : str, mountpoint : str): 

131 # Create a fake file to be opened in case it does not exists yet 

132 handler = FileHandler(url) 

133 # Create the mount point in case it does not exist yet 

134 if not os.path.exists(mountpoint): 

135 open(mountpoint, 'w').close() 

136 # Mount the handler in the mount point 

137 FUSE(handler, mountpoint, nothreads=True, foreground=True) 

138 

139if __name__ == '__main__': 

140 # Example usage 

141 url = 'https://cineca.mddbr.eu/api/rest/v1/projects/A0001/files/structure.pdb' 

142 mountpoint = '/home/rchaves/Downloads/structure.pdb' 

143 mount(url, mountpoint)