1   
  2   
  3   
  4   
  5   
  6   
  7   
  8   
  9   
 10   
 11   
 12   
 13   
 14   
 15   
 16   
 17   
 18   
 19  """ 
 20  L{SFTPFile} 
 21  """ 
 22   
 23  from binascii import hexlify 
 24  import socket 
 25  import threading 
 26  import time 
 27   
 28  from paramiko.common import * 
 29  from paramiko.sftp import * 
 30  from paramiko.file import BufferedFile 
 31  from paramiko.sftp_attr import SFTPAttributes 
 32   
 33   
 35      """ 
 36      Proxy object for a file on the remote server, in client mode SFTP. 
 37      """ 
 38   
 39       
 40       
 41      MAX_REQUEST_SIZE = 32768 
 42   
 43 -    def __init__(self, sftp, handle, mode='r', bufsize=-1): 
  44          BufferedFile.__init__(self) 
 45          self.sftp = sftp 
 46          self.handle = handle 
 47          BufferedFile._set_mode(self, mode, bufsize) 
 48          self.pipelined = False 
 49          self._prefetching = False 
 50          self._prefetch_done = False 
 51          self._prefetch_data = {} 
 52          self._prefetch_reads = [] 
 53          self._saved_exception = None 
  54   
 56          self._close(async=True) 
  57       
 59          self._close(async=False) 
  60           
 61 -    def _close(self, async=False): 
  62           
 63           
 64           
 65           
 66           
 67           
 68           
 69          if self._closed: 
 70              return 
 71          self.sftp._log(DEBUG, 'close(%s)' % hexlify(self.handle)) 
 72          if self.pipelined: 
 73              self.sftp._finish_responses(self) 
 74          BufferedFile.close(self) 
 75          try: 
 76              if async: 
 77                   
 78                  self.sftp._async_request(type(None), CMD_CLOSE, self.handle) 
 79              else: 
 80                  self.sftp._request(CMD_CLOSE, self.handle) 
 81          except EOFError: 
 82               
 83              pass 
 84          except (IOError, socket.error): 
 85               
 86              pass 
  87   
 89          k = [i for i in self._prefetch_reads if i[0] <= offset] 
 90          if len(k) == 0: 
 91              return False 
 92          k.sort(lambda x, y: cmp(x[0], y[0])) 
 93          buf_offset, buf_size = k[-1] 
 94          if buf_offset + buf_size <= offset: 
 95               
 96              return False 
 97          if buf_offset + buf_size >= offset + size: 
 98               
 99              return True 
100           
101          return self._data_in_prefetch_requests(buf_offset + buf_size, offset + size - buf_offset - buf_size) 
 102       
104          """ 
105          if a block of data is present in the prefetch buffers, at the given 
106          offset, return the offset of the relevant prefetch buffer.  otherwise, 
107          return None.  this guarantees nothing about the number of bytes 
108          collected in the prefetch buffer so far. 
109          """ 
110          k = [i for i in self._prefetch_data.keys() if i <= offset] 
111          if len(k) == 0: 
112              return None 
113          index = max(k) 
114          buf_offset = offset - index 
115          if buf_offset >= len(self._prefetch_data[index]): 
116               
117              return None 
118          return index 
 119           
121          """ 
122          read data out of the prefetch buffer, if possible.  if the data isn't 
123          in the buffer, return None.  otherwise, behaves like a normal read. 
124          """ 
125           
126          while True: 
127              offset = self._data_in_prefetch_buffers(self._realpos) 
128              if offset is not None: 
129                  break 
130              if self._prefetch_done or self._closed: 
131                  break 
132              self.sftp._read_response() 
133              self._check_exception() 
134          if offset is None: 
135              self._prefetching = False 
136              return None 
137          prefetch = self._prefetch_data[offset] 
138          del self._prefetch_data[offset] 
139           
140          buf_offset = self._realpos - offset 
141          if buf_offset > 0: 
142              self._prefetch_data[offset] = prefetch[:buf_offset] 
143              prefetch = prefetch[buf_offset:] 
144          if size < len(prefetch): 
145              self._prefetch_data[self._realpos + size] = prefetch[size:] 
146              prefetch = prefetch[:size] 
147          return prefetch 
 148           
150          size = min(size, self.MAX_REQUEST_SIZE) 
151          if self._prefetching: 
152              data = self._read_prefetch(size) 
153              if data is not None: 
154                  return data 
155          t, msg = self.sftp._request(CMD_READ, self.handle, long(self._realpos), int(size)) 
156          if t != CMD_DATA: 
157              raise SFTPError('Expected data') 
158          return msg.get_string() 
 159   
161           
162          chunk = min(len(data), self.MAX_REQUEST_SIZE) 
163          req = self.sftp._async_request(type(None), CMD_WRITE, self.handle, long(self._realpos), str(data[:chunk])) 
164          if not self.pipelined or self.sftp.sock.recv_ready(): 
165              t, msg = self.sftp._read_response(req) 
166              if t != CMD_STATUS: 
167                  raise SFTPError('Expected status') 
168               
169          return chunk 
 170   
172          """ 
173          Set a timeout on read/write operations on the underlying socket or 
174          ssh L{Channel}. 
175   
176          @see: L{Channel.settimeout} 
177          @param timeout: seconds to wait for a pending read/write operation 
178              before raising C{socket.timeout}, or C{None} for no timeout 
179          @type timeout: float 
180          """ 
181          self.sftp.sock.settimeout(timeout) 
 182   
184          """ 
185          Returns the timeout in seconds (as a float) associated with the socket 
186          or ssh L{Channel} used for this file. 
187   
188          @see: L{Channel.gettimeout} 
189          @rtype: float 
190          """ 
191          return self.sftp.sock.gettimeout() 
 192   
194          """ 
195          Set blocking or non-blocking mode on the underiying socket or ssh 
196          L{Channel}. 
197   
198          @see: L{Channel.setblocking} 
199          @param blocking: 0 to set non-blocking mode; non-0 to set blocking 
200              mode. 
201          @type blocking: int 
202          """ 
203          self.sftp.sock.setblocking(blocking) 
 204   
205 -    def seek(self, offset, whence=0): 
 206          self.flush() 
207          if whence == self.SEEK_SET: 
208              self._realpos = self._pos = offset 
209          elif whence == self.SEEK_CUR: 
210              self._pos += offset 
211              self._realpos = self._pos 
212          else: 
213              self._realpos = self._pos = self._get_size() + offset 
214          self._rbuffer = '' 
 215   
217          """ 
218          Retrieve information about this file from the remote system.  This is 
219          exactly like L{SFTP.stat}, except that it operates on an already-open 
220          file. 
221   
222          @return: an object containing attributes about this file. 
223          @rtype: SFTPAttributes 
224          """ 
225          t, msg = self.sftp._request(CMD_FSTAT, self.handle) 
226          if t != CMD_ATTRS: 
227              raise SFTPError('Expected attributes') 
228          return SFTPAttributes._from_msg(msg) 
 229   
231          """ 
232          Change the mode (permissions) of this file.  The permissions are 
233          unix-style and identical to those used by python's C{os.chmod} 
234          function. 
235   
236          @param mode: new permissions 
237          @type mode: int 
238          """ 
239          self.sftp._log(DEBUG, 'chmod(%s, %r)' % (hexlify(self.handle), mode)) 
240          attr = SFTPAttributes() 
241          attr.st_mode = mode 
242          self.sftp._request(CMD_FSETSTAT, self.handle, attr) 
 243           
244 -    def chown(self, uid, gid): 
 245          """ 
246          Change the owner (C{uid}) and group (C{gid}) of this file.  As with 
247          python's C{os.chown} function, you must pass both arguments, so if you 
248          only want to change one, use L{stat} first to retrieve the current 
249          owner and group. 
250   
251          @param uid: new owner's uid 
252          @type uid: int 
253          @param gid: new group id 
254          @type gid: int 
255          """ 
256          self.sftp._log(DEBUG, 'chown(%s, %r, %r)' % (hexlify(self.handle), uid, gid)) 
257          attr = SFTPAttributes() 
258          attr.st_uid, attr.st_gid = uid, gid 
259          self.sftp._request(CMD_FSETSTAT, self.handle, attr) 
 260   
262          """ 
263          Set the access and modified times of this file.  If 
264          C{times} is C{None}, then the file's access and modified times are set 
265          to the current time.  Otherwise, C{times} must be a 2-tuple of numbers, 
266          of the form C{(atime, mtime)}, which is used to set the access and 
267          modified times, respectively.  This bizarre API is mimicked from python 
268          for the sake of consistency -- I apologize. 
269   
270          @param times: C{None} or a tuple of (access time, modified time) in 
271              standard internet epoch time (seconds since 01 January 1970 GMT) 
272          @type times: tuple(int) 
273          """ 
274          if times is None: 
275              times = (time.time(), time.time()) 
276          self.sftp._log(DEBUG, 'utime(%s, %r)' % (hexlify(self.handle), times)) 
277          attr = SFTPAttributes() 
278          attr.st_atime, attr.st_mtime = times 
279          self.sftp._request(CMD_FSETSTAT, self.handle, attr) 
 280   
282          """ 
283          Change the size of this file.  This usually extends 
284          or shrinks the size of the file, just like the C{truncate()} method on 
285          python file objects. 
286           
287          @param size: the new size of the file 
288          @type size: int or long 
289          """ 
290          self.sftp._log(DEBUG, 'truncate(%s, %r)' % (hexlify(self.handle), size)) 
291          attr = SFTPAttributes() 
292          attr.st_size = size 
293          self.sftp._request(CMD_FSETSTAT, self.handle, attr) 
 294       
295 -    def check(self, hash_algorithm, offset=0, length=0, block_size=0): 
 296          """ 
297          Ask the server for a hash of a section of this file.  This can be used 
298          to verify a successful upload or download, or for various rsync-like 
299          operations. 
300           
301          The file is hashed from C{offset}, for C{length} bytes.  If C{length} 
302          is 0, the remainder of the file is hashed.  Thus, if both C{offset} 
303          and C{length} are zero, the entire file is hashed. 
304           
305          Normally, C{block_size} will be 0 (the default), and this method will 
306          return a byte string representing the requested hash (for example, a 
307          string of length 16 for MD5, or 20 for SHA-1).  If a non-zero 
308          C{block_size} is given, each chunk of the file (from C{offset} to 
309          C{offset + length}) of C{block_size} bytes is computed as a separate 
310          hash.  The hash results are all concatenated and returned as a single 
311          string. 
312           
313          For example, C{check('sha1', 0, 1024, 512)} will return a string of 
314          length 40.  The first 20 bytes will be the SHA-1 of the first 512 bytes 
315          of the file, and the last 20 bytes will be the SHA-1 of the next 512 
316          bytes. 
317           
318          @param hash_algorithm: the name of the hash algorithm to use (normally 
319              C{"sha1"} or C{"md5"}) 
320          @type hash_algorithm: str 
321          @param offset: offset into the file to begin hashing (0 means to start 
322              from the beginning) 
323          @type offset: int or long 
324          @param length: number of bytes to hash (0 means continue to the end of 
325              the file) 
326          @type length: int or long 
327          @param block_size: number of bytes to hash per result (must not be less 
328              than 256; 0 means to compute only one hash of the entire segment) 
329          @type block_size: int 
330          @return: string of bytes representing the hash of each block, 
331              concatenated together 
332          @rtype: str 
333           
334          @note: Many (most?) servers don't support this extension yet. 
335           
336          @raise IOError: if the server doesn't support the "check-file" 
337              extension, or possibly doesn't support the hash algorithm 
338              requested 
339               
340          @since: 1.4 
341          """ 
342          t, msg = self.sftp._request(CMD_EXTENDED, 'check-file', self.handle, 
343                                      hash_algorithm, long(offset), long(length), block_size) 
344          ext = msg.get_string() 
345          alg = msg.get_string() 
346          data = msg.get_remainder() 
347          return data 
 348       
350          """ 
351          Turn on/off the pipelining of write operations to this file.  When 
352          pipelining is on, paramiko won't wait for the server response after 
353          each write operation.  Instead, they're collected as they come in. 
354          At the first non-write operation (including L{close}), all remaining 
355          server responses are collected.  This means that if there was an error 
356          with one of your later writes, an exception might be thrown from 
357          within L{close} instead of L{write}. 
358           
359          By default, files are I{not} pipelined. 
360           
361          @param pipelined: C{True} if pipelining should be turned on for this 
362              file; C{False} otherwise 
363          @type pipelined: bool 
364           
365          @since: 1.5 
366          """ 
367          self.pipelined = pipelined 
 368       
370          """ 
371          Pre-fetch the remaining contents of this file in anticipation of 
372          future L{read} calls.  If reading the entire file, pre-fetching can 
373          dramatically improve the download speed by avoiding roundtrip latency. 
374          The file's contents are incrementally buffered in a background thread. 
375           
376          The prefetched data is stored in a buffer until read via the L{read} 
377          method.  Once data has been read, it's removed from the buffer.  The 
378          data may be read in a random order (using L{seek}); chunks of the 
379          buffer that haven't been read will continue to be buffered. 
380   
381          @since: 1.5.1 
382          """ 
383          size = self.stat().st_size 
384           
385          chunks = [] 
386          n = self._realpos 
387          while n < size: 
388              chunk = min(self.MAX_REQUEST_SIZE, size - n) 
389              chunks.append((n, chunk)) 
390              n += chunk 
391          if len(chunks) > 0: 
392              self._start_prefetch(chunks) 
 393       
394 -    def readv(self, chunks): 
 395          """ 
396          Read a set of blocks from the file by (offset, length).  This is more 
397          efficient than doing a series of L{seek} and L{read} calls, since the 
398          prefetch machinery is used to retrieve all the requested blocks at 
399          once. 
400           
401          @param chunks: a list of (offset, length) tuples indicating which 
402              sections of the file to read 
403          @type chunks: list(tuple(long, int)) 
404          @return: a list of blocks read, in the same order as in C{chunks} 
405          @rtype: list(str) 
406           
407          @since: 1.5.4 
408          """ 
409          self.sftp._log(DEBUG, 'readv(%s, %r)' % (hexlify(self.handle), chunks)) 
410   
411          read_chunks = [] 
412          for offset, size in chunks: 
413               
414              if self._data_in_prefetch_buffers(offset) or self._data_in_prefetch_requests(offset, size): 
415                  continue 
416   
417               
418              while size > 0: 
419                  chunk_size = min(size, self.MAX_REQUEST_SIZE) 
420                  read_chunks.append((offset, chunk_size)) 
421                  offset += chunk_size 
422                  size -= chunk_size 
423   
424          self._start_prefetch(read_chunks) 
425           
426          for x in chunks: 
427              self.seek(x[0]) 
428              yield self.read(x[1]) 
 429       
430   
431       
432   
433   
435          try: 
436              return self.stat().st_size 
437          except: 
438              return 0 
 439   
441          self._prefetching = True 
442          self._prefetch_done = False 
443          self._prefetch_reads.extend(chunks) 
444   
445          t = threading.Thread(target=self._prefetch_thread, args=(chunks,)) 
446          t.setDaemon(True) 
447          t.start() 
 448           
450           
451           
452          for offset, length in chunks: 
453              self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length)) 
 454   
456          if t == CMD_STATUS: 
457               
458              try: 
459                  self.sftp._convert_status(msg) 
460              except Exception, x: 
461                  self._saved_exception = x 
462              return 
463          if t != CMD_DATA: 
464              raise SFTPError('Expected data') 
465          data = msg.get_string() 
466          offset, length = self._prefetch_reads.pop(0) 
467          self._prefetch_data[offset] = data 
468          if len(self._prefetch_reads) == 0: 
469              self._prefetch_done = True 
 470       
472          "if there's a saved exception, raise & clear it" 
473          if self._saved_exception is not None: 
474              x = self._saved_exception 
475              self._saved_exception = None 
476              raise x 
  477