-
Notifications
You must be signed in to change notification settings - Fork 12
/
fuse_rsync.py
349 lines (287 loc) · 12 KB
/
fuse_rsync.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
#!/usr/bin/env python
#Copyright (c) 2014 Jonas Zaddach
#Licensed under the MIT License (https://github.com/zaddach/fuse-rsync/blob/master/LICENSE)
import os
import sys
import errno
import optparse
import logging
import subprocess
import stat
import datetime
import time
import re
import fuse
from tempfile import mkstemp
from threading import Lock
fuse.fuse_python_api = (0, 2)
log = logging.getLogger('fuse_rsync')
class RsyncModule():
"""
This class implements access to an Rsync module.
"""
def __init__(self, host, module, user = None, password = None):
self._environment = os.environ.copy()
self._remote_url = "rsync://"
if not user is None:
self._remote_url += user + "@"
self._remote_url += host + "/" + module
if not password is None:
self._environment['RSYNC_PASSWORD'] = password
self._attr_cache = {}
def _parse_attrs(self, attrs):
"""
Parse the textual representation of file attributes to binary representation.
"""
result = 0
if attrs[0] == 'd':
result |= stat.S_IFDIR
elif attrs[0] == 'l':
result |= stat.S_IFLNK
elif attrs[0] == '-':
result |= stat.S_IFREG
else:
assert(False)
for i in range(0, 3):
val = 0
if 'r' in attrs[1 + 3 * i: 4 + 3 * i]:
val |= 4
if 'w' in attrs[1 + 3 * i: 4 + 3 * i]:
val |= 2
if 'x' in attrs[1 + 3 * i: 4 + 3 * i]:
val |= 1
result |= val << ((2 - i) * 3)
return result
def list(self, path = '/'):
"""
List files contained in directory __path__.
Returns a list of dictionaries with keys *attrs* (numerical attribute
representation), *size* (file size), *timestamp* (File's atime timestamp
in a datetime object) and *filename* (The file's name).
"""
# See http://stackoverflow.com/questions/10323060/printing-file-permissions-like-ls-l-using-stat2-in-c for modes
RE_LINE = re.compile("^([ldcbps-]([r-][w-][x-]){3})\s+([0-9]+)\s+([0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) (.*)$")
remote_url = self._remote_url + path
try:
cmdline = ["rsync", "--list-only", remote_url]
log.debug("executing %s", " ".join(cmdline))
output = subprocess.check_output(["rsync", "--list-only", remote_url], env = self._environment)
listing = []
for line in output.decode(encoding = 'iso-8859-1').split("\n"):
match = RE_LINE.match(line)
if not match:
continue
listing.append({
"attrs": self._parse_attrs(match.group(1)),
"size": int(match.group(3)),
"timestamp": datetime.datetime.strptime(match.group(4), "%Y/%m/%d %H:%M:%S"),
"filename": match.group(5)
})
return listing
except subprocess.CalledProcessError as err:
if err.returncode == 23:
return []
raise err
def copy(self, remotepath = '/', localpath = None):
"""
Copy a file from the remote rsync module to the local filesystem.
If no local destination is specified in __localpath__, a temporary
file is created and its filename returned. The temporary file has
to be deleted by the caller.
"""
remote_url = self._remote_url + remotepath
if localpath is None:
(file, localpath) = mkstemp()
os.close(file)
cmdline = ["rsync", "--copy-links", remote_url, localpath]
log.debug("executing %s", " ".join(cmdline))
subprocess.check_call(cmdline, env = self._environment)
return localpath
class FuseRsyncFileInfo(fuse.FuseFileInfo):
"""
Encapsulates the file handle for an opened file.
"""
def __init__(self, handle, **kw):
super(FuseRsyncFileInfo, self).__init__(**kw)
self.keep = True
self.handle = handle
class FuseRsync(fuse.Fuse):
"""
The implementation of the FUSE filesystem.
"""
def __init__(self, *args, **kw):
self.host = None
self.module = None
self.user = None
self.password = None
self.path = "/"
self._attr_cache = {}
self._file_cache = {}
self._file_cache_lock = Lock()
fuse.Fuse.__init__(self, *args, **kw)
self.parser.add_option(mountopt = 'user', default = None, help = "Rsync user on the remote host")
self.parser.add_option(mountopt = 'password', type = str, default = None, help = "Rsync password on the remote host")
self.parser.add_option(mountopt = 'host', type = str, help = "Rsync remote host")
self.parser.add_option(mountopt = 'module', type = str, help = "Rsync module on remote host")
self.parser.add_option(mountopt = 'path', type = str, default = "/", help = "Rsync path in module on remote host that is supposed to be the root point")
# Helpers
# =======
def _full_path(self, partial):
if partial.startswith("/"):
partial = partial[1:]
path = os.path.join(self.path, partial)
return path
def init(self):
options = self.cmdline[0]
log.debug("Invoked fsinit() with host=%s, module=%s, user=%s, password=%s", options.host, options.module, options.user, options.password)
self._rsync = RsyncModule(options.host, options.module, options.user, options.password)
# Filesystem methods
# ==================
#def access(self, path, mode):
#full_path = self._full_path(path)
#if not os.access(full_path, mode):
#raise FuseOSError(errno.EACCES)
#def chmod(self, path, mode):
#full_path = self._full_path(path)
#return os.chmod(full_path, mode)
#def chown(self, path, uid, gid):
#full_path = self._full_path(path)
#return os.chown(full_path, uid, gid)
def getattr(self, path, fh=None):
try:
log.debug("Invoked getattr('%s')", path)
path = self._full_path(path)
st = fuse.Stat()
if path == "/":
st.st_atime = int(time.time())
st.st_ctime = int(time.time())
st.st_mode = stat.S_IFDIR | 0555
st.st_mtime = int(time.time())
st.st_nlink = 2
st.st_uid = os.geteuid()
st.st_gid = os.getegid()
return st
if path in self._attr_cache:
info = self._attr_cache[path]
else:
listing = self._rsync.list(path)
if len(listing) != 1:
log.warn("Found none or several files for path")
return -errno.ENOENT
info = listing[0]
self._attr_cache[path] = info
timestamp = (info["timestamp"] - datetime.datetime(1970,1,1)).total_seconds()
st.st_atime = timestamp
st.st_ctime = timestamp
st.st_uid = os.geteuid()
st.st_gid = os.getegid()
if info["attrs"] & stat.S_IFDIR:
st.st_mode = stat.S_IFDIR | 0555
else:
st.st_mode = stat.S_IFREG | 0444
st.st_mtime = timestamp
st.st_nlink = 1
st.st_size = info["size"]
return st
except Exception as ex:
log.exception("while doing getattr")
return -errno.ENOENT
def readdir(self, path, offset):
try:
if not path.endswith("/"):
path += "/"
log.debug("Invoked readdir('%s')", path)
full_path = self._full_path(path)
yield fuse.Direntry('.')
yield fuse.Direntry('..')
for dirent in self._rsync.list(full_path):
if dirent["filename"] == ".":
continue
self._attr_cache[path + dirent["filename"]] = dirent
yield fuse.Direntry(str(dirent["filename"]))
except Exception as ex:
log.exception("While doing readdir")
#def readlink(self, path):
#pathname = os.readlink(self._full_path(path))
#if pathname.startswith("/"):
## Path name is absolute, sanitize it.
#return os.path.relpath(pathname, self.root)
#else:
#return pathname
#def mknod(self, path, mode, dev):
#return os.mknod(self._full_path(path), mode, dev)
#def rmdir(self, path):
#full_path = self._full_path(path)
#return os.rmdir(full_path)
#def mkdir(self, path, mode):
#return os.mkdir(self._full_path(path), mode)
#def statfs(self, path):
#full_path = self._full_path(path)
#stv = os.statvfs(full_path)
#return dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree',
#'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag',
#'f_frsize', 'f_namemax'))
#def unlink(self, path):
#return os.unlink(self._full_path(path))
#def symlink(self, target, name):
#return os.symlink(self._full_path(target), self._full_path(name))
#def rename(self, old, new):
#return os.rename(self._full_path(old), self._full_path(new))
#def link(self, target, name):
#return os.link(self._full_path(target), self._full_path(name))
#def utimens(self, path, times=None):
#return os.utime(self._full_path(path), times)
## File methods
## ============
def open(self, path, flags):
log.debug("invoking open(%s, %d)", path, flags)
full_path = self._full_path(path)
if flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR) != os.O_RDONLY:
return -errno.EACCES
with self._file_cache_lock:
if not path in self._file_cache:
localfile = self._rsync.copy(full_path)
self._file_cache[path] = {"refcount": 1, "localpath": localfile}
else:
self._file_cache[path]["refcount"] += 1
localfile = self._file_cache[path]["localpath"]
handle = os.open(localfile, os.O_RDONLY)
log.debug("Created file handle %d", handle)
return FuseRsyncFileInfo(handle)
#def create(self, path, mode, fi=None):
#full_path = self._full_path(path)
#return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode)
def read(self, path, length, offset, fh):
log.debug("invoking read(%s, %d, %d, %d)", path, length, offset, fh.handle)
os.lseek(fh.handle, offset, os.SEEK_SET)
return os.read(fh.handle, length)
#def write(self, path, buf, offset, fh):
#os.lseek(fh, offset, os.SEEK_SET)
#return os.write(fh, buf)
#def truncate(self, path, length, fh=None):
#full_path = self._full_path(path)
#with open(full_path, 'r+') as f:
#f.truncate(length)
#def flush(self, path, fh):
#return os.fsync(fh)
def release(self, path, dummy, fh):
log.debug("invoking release(%s, %d, %d)", path, dummy, fh.handle)
os.close(fh.handle)
with self._file_cache_lock:
self._file_cache[path]["refcount"] -= 1
if self._file_cache[path]["refcount"] <= 0:
localfile = self._file_cache[path]["localpath"]
del self._file_cache[path]
os.unlink(localfile)
#def fsync(self, path, fdatasync, fh):
#return self.flush(path, fh)
if __name__ == '__main__':
fs = FuseRsync()
fs.parse(errex=1)
#TODO: Below is hacky, find properly parsed debug attribute
if '-d' in sys.argv:
logging.basicConfig(level = logging.DEBUG)
else:
logging.basicConfig(level = logging.ERROR)
fs.init()
fs.main()