1  """ 
  2  connection operations 
  3   
  4  Connection instances are used to communicate with the remote service at 
  5  the account level creating, listing and deleting Containers, and returning 
  6  Container instances. 
  7   
  8  See COPYING for license information. 
  9  """ 
 10   
 11  import  socket 
 12  import  os 
 13  from    urllib    import urlencode 
 14  from    httplib   import HTTPSConnection, HTTPConnection, HTTPException 
 15  from    container import Container, ContainerResults 
 16  from    utils     import unicode_quote, parse_url, THTTPConnection, THTTPSConnection 
 17  from    errors    import ResponseError, NoSuchContainer, ContainerNotEmpty, \ 
 18                           InvalidContainerName, CDNNotEnabled, ContainerExists 
 19  from    Queue     import Queue, Empty, Full 
 20  from    time      import time 
 21  import  consts 
 22  from    authentication import Authentication 
 23  from    fjson     import json_loads 
 24  from    sys       import version_info 
 25   
 26   
 27   
 28   
 29   
 31      """ 
 32      Manages the connection to the storage system and serves as a factory 
 33      for Container instances. 
 34   
 35      @undocumented: cdn_connect 
 36      @undocumented: http_connect 
 37      @undocumented: cdn_request 
 38      @undocumented: make_request 
 39      @undocumented: _check_container_name 
 40      """ 
 41   
 42 -    def __init__(self, username=None, api_key=None, timeout=5, **kwargs): 
  43          """ 
 44          Accepts keyword arguments for Mosso username and api key. 
 45          Optionally, you can omit these keywords and supply an 
 46          Authentication object using the auth keyword. Setting the argument 
 47          servicenet to True will make use of Rackspace servicenet network. 
 48   
 49          @type username: str 
 50          @param username: a Mosso username 
 51          @type api_key: str 
 52          @param api_key: a Mosso API key 
 53          @type servicenet: bool 
 54          @param servicenet: Use Rackspace servicenet to access Cloud Files. 
 55          @type cdn_log_retention: bool 
 56          @param cdn_log_retention: set logs retention for this cdn enabled 
 57          container. 
 58          """ 
 59          self.cdn_enabled = False 
 60          self.cdn_args = None 
 61          self.connection_args = None 
 62          self.cdn_connection = None 
 63          self.connection = None 
 64          self.token = None 
 65          self.debuglevel = int(kwargs.get('debuglevel', 0)) 
 66          self.servicenet = kwargs.get('servicenet', False) 
 67          self.user_agent = kwargs.get('useragent', consts.user_agent) 
 68          self.timeout = timeout 
 69   
 70           
 71           
 72          if not 'servicenet' in kwargs \ 
 73                  and 'RACKSPACE_SERVICENET' in os.environ: 
 74              self.servicenet = True 
 75   
 76          self.auth = 'auth' in kwargs and kwargs['auth'] or None 
 77   
 78          if not self.auth: 
 79              authurl = kwargs.get('authurl', consts.us_authurl) 
 80              if username and api_key and authurl: 
 81                  self.auth = Authentication(username, api_key, authurl=authurl, 
 82                              useragent=self.user_agent, timeout=self.timeout) 
 83              else: 
 84                  raise TypeError("Incorrect or invalid arguments supplied") 
 85          self._authenticate() 
  87          """ 
 88          Authenticate and setup this instance with the values returned. 
 89          """ 
 90          (url, self.cdn_url, self.token) = self.auth.authenticate() 
 91          url = self._set_storage_url(url) 
 92          self.connection_args = parse_url(url) 
 93   
 94          if version_info[0] <= 2 and version_info[1] < 6: 
 95              self.conn_class = self.connection_args[3] and THTTPSConnection or \ 
 96                                                                THTTPConnection 
 97          else: 
 98              self.conn_class = self.connection_args[3] and HTTPSConnection or \ 
 99                                                                HTTPConnection 
100          self.http_connect() 
101          if self.cdn_url: 
102              self.cdn_connect() 
 103   
105          if self.servicenet: 
106              return "https://snet-%s" % url.replace("https://", "") 
107          return url 
 108   
110          """ 
111          Setup the http connection instance for the CDN service. 
112          """ 
113          (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url) 
114          self.cdn_connection = self.conn_class(host, port, timeout=self.timeout) 
115          self.cdn_enabled = True 
 116   
118          """ 
119          Setup the http connection instance. 
120          """ 
121          (host, port, self.uri, is_ssl) = self.connection_args 
122          self.connection = self.conn_class(host, port=port, \ 
123                                                timeout=self.timeout) 
124          self.connection.set_debuglevel(self.debuglevel) 
 125   
126 -    def cdn_request(self, method, path=[], data='', hdrs=None): 
 127          """ 
128          Given a method (i.e. GET, PUT, POST, etc), a path, data, header and 
129          metadata dicts, performs an http request against the CDN service. 
130          """ 
131          if not self.cdn_enabled: 
132              raise CDNNotEnabled() 
133   
134          path = '/%s/%s' % \ 
135                   (self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path])) 
136          headers = {'Content-Length': str(len(data)), 
137                     'User-Agent': self.user_agent, 
138                     'X-Auth-Token': self.token} 
139          if isinstance(hdrs, dict): 
140              headers.update(hdrs) 
141   
142          def retry_request(): 
143              '''Re-connect and re-try a failed request once''' 
144              self.cdn_connect() 
145              self.cdn_connection.request(method, path, data, headers) 
146              return self.cdn_connection.getresponse() 
 147   
148          try: 
149              self.cdn_connection.request(method, path, data, headers) 
150              response = self.cdn_connection.getresponse() 
151          except (socket.error, IOError, HTTPException): 
152              response = retry_request() 
153          if response.status == 401: 
154              self._authenticate() 
155              headers['X-Auth-Token'] = self.token 
156              response = retry_request() 
157   
158          return response 
 159   
160 -    def make_request(self, method, path=[], data='', hdrs=None, parms=None): 
 161          """ 
162          Given a method (i.e. GET, PUT, POST, etc), a path, data, header and 
163          metadata dicts, and an optional dictionary of query parameters, 
164          performs an http request. 
165          """ 
166          path = '/%s/%s' % \ 
167                   (self.uri.rstrip('/'), '/'.join([unicode_quote(i) for i in path])) 
168   
169          if isinstance(parms, dict) and parms: 
170              path = '%s?%s' % (path, urlencode(parms)) 
171   
172          headers = {'Content-Length': str(len(data)), 
173                     'User-Agent': self.user_agent, 
174                     'X-Auth-Token': self.token} 
175          isinstance(hdrs, dict) and headers.update(hdrs) 
176   
177          def retry_request(): 
178              '''Re-connect and re-try a failed request once''' 
179              self.http_connect() 
180              self.connection.request(method, path, data, headers) 
181              return self.connection.getresponse() 
 182   
183          try: 
184              self.connection.request(method, path, data, headers) 
185              response = self.connection.getresponse() 
186          except (socket.error, IOError, HTTPException): 
187              response = retry_request() 
188          if response.status == 401: 
189              self._authenticate() 
190              headers['X-Auth-Token'] = self.token 
191              response = retry_request() 
192   
193          return response 
194   
196          """ 
197          Return tuple for number of containers, total bytes in the account and account metadata 
198   
199          >>> connection.get_info() 
200          (5, 2309749) 
201   
202          @rtype: tuple 
203          @return: a tuple containing the number of containers, total bytes 
204                   used by the account and a dictionary containing account metadata 
205          """ 
206          response = self.make_request('HEAD') 
207          count = size = None 
208          metadata = {} 
209          for hdr in response.getheaders(): 
210              if hdr[0].lower() == 'x-account-container-count': 
211                  try: 
212                      count = int(hdr[1]) 
213                  except ValueError: 
214                      count = 0 
215              if hdr[0].lower() == 'x-account-bytes-used': 
216                  try: 
217                      size = int(hdr[1]) 
218                  except ValueError: 
219                      size = 0 
220              if hdr[0].lower().startswith('x-account-meta-'): 
221                  metadata[hdr[0].lower()[15:]] = hdr[1] 
222          buff = response.read() 
223          if (response.status < 200) or (response.status > 299): 
224              raise ResponseError(response.status, response.reason) 
225          return (count, size, metadata) 
 226   
240   
242          if not container_name or \ 
243                  '/' in container_name or \ 
244                  len(container_name) > consts.container_name_limit: 
245              raise InvalidContainerName(container_name) 
 246   
248          """ 
249          Given a container name, returns a L{Container} item, creating a new 
250          Container if one does not already exist. 
251   
252          >>> connection.create_container('new_container') 
253          <cloudfiles.container.Container object at 0xb77d628c> 
254   
255          @param container_name: name of the container to create 
256          @type container_name: str 
257          @param error_on_existing: raise ContainerExists if container already 
258          exists 
259          @type error_on_existing: bool 
260          @rtype: L{Container} 
261          @return: an object representing the newly created container 
262          """ 
263          self._check_container_name(container_name) 
264   
265          response = self.make_request('PUT', [container_name]) 
266          buff = response.read() 
267          if (response.status < 200) or (response.status > 299): 
268              raise ResponseError(response.status, response.reason) 
269          if error_on_existing and (response.status == 202): 
270              raise ContainerExists(container_name) 
271          return Container(self, container_name) 
 272   
274          """ 
275          Given a container name, delete it. 
276   
277          >>> connection.delete_container('old_container') 
278   
279          @param container_name: name of the container to delete 
280          @type container_name: str 
281          """ 
282          if isinstance(container_name, Container): 
283              container_name = container_name.name 
284          self._check_container_name(container_name) 
285   
286          response = self.make_request('DELETE', [container_name]) 
287          response.read() 
288   
289          if (response.status == 409): 
290              raise ContainerNotEmpty(container_name) 
291          elif (response.status == 404): 
292              raise NoSuchContainer 
293          elif (response.status < 200) or (response.status > 299): 
294              raise ResponseError(response.status, response.reason) 
295   
296          if self.cdn_enabled: 
297              response = self.cdn_request('POST', [container_name], 
298                                  hdrs={'X-CDN-Enabled': 'False'}) 
 299   
301          """ 
302          Returns a Container item result set. 
303   
304          >>> connection.get_all_containers() 
305          ContainerResults: 4 containers 
306          >>> print ', '.join([container.name for container in 
307                               connection.get_all_containers()]) 
308          new_container, old_container, pictures, music 
309   
310          @rtype: L{ContainerResults} 
311          @return: an iterable set of objects representing all containers on the 
312                   account 
313          @param limit: number of results to return, up to 10,000 
314          @type limit: int 
315          @param marker: return only results whose name is greater than "marker" 
316          @type marker: str 
317          """ 
318          if limit: 
319              parms['limit'] = limit 
320          if marker: 
321              parms['marker'] = marker 
322          return ContainerResults(self, self.list_containers_info(**parms)) 
 323   
325          """ 
326          Return a single Container item for the given Container. 
327   
328          >>> connection.get_container('old_container') 
329          <cloudfiles.container.Container object at 0xb77d628c> 
330          >>> container = connection.get_container('old_container') 
331          >>> container.size_used 
332          23074 
333   
334          @param container_name: name of the container to create 
335          @type container_name: str 
336          @rtype: L{Container} 
337          @return: an object representing the container 
338          """ 
339          self._check_container_name(container_name) 
340   
341          response = self.make_request('HEAD', [container_name]) 
342          count = size = None 
343          metadata = {} 
344          for hdr in response.getheaders(): 
345              if hdr[0].lower() == 'x-container-object-count': 
346                  try: 
347                      count = int(hdr[1]) 
348                  except ValueError: 
349                      count = 0 
350              if hdr[0].lower() == 'x-container-bytes-used': 
351                  try: 
352                      size = int(hdr[1]) 
353                  except ValueError: 
354                      size = 0 
355              if hdr[0].lower().startswith('x-container-meta-'): 
356                  metadata[hdr[0].lower()[17:]] = hdr[1] 
357          buff = response.read() 
358          if response.status == 404: 
359              raise NoSuchContainer(container_name) 
360          if (response.status < 200) or (response.status > 299): 
361              raise ResponseError(response.status, response.reason) 
362          return Container(self, container_name, count, size, metadata) 
 363   
365          """ 
366          Returns a list of containers that have been published to the CDN. 
367   
368          >>> connection.list_public_containers() 
369          ['container1', 'container2', 'container3'] 
370   
371          @rtype: list(str) 
372          @return: a list of all CDN-enabled container names as strings 
373          """ 
374          response = self.cdn_request('GET', ['']) 
375          if (response.status < 200) or (response.status > 299): 
376              buff = response.read() 
377              raise ResponseError(response.status, response.reason) 
378          return response.read().splitlines() 
 379   
381          """ 
382          Returns a list of Containers, including object count and size. 
383   
384          >>> connection.list_containers_info() 
385          [{u'count': 510, u'bytes': 2081717, u'name': u'new_container'}, 
386           {u'count': 12, u'bytes': 23074, u'name': u'old_container'}, 
387           {u'count': 0, u'bytes': 0, u'name': u'container1'}, 
388           {u'count': 0, u'bytes': 0, u'name': u'container2'}, 
389           {u'count': 0, u'bytes': 0, u'name': u'container3'}, 
390           {u'count': 3, u'bytes': 2306, u'name': u'test'}] 
391   
392          @rtype: list({"name":"...", "count":..., "bytes":...}) 
393          @return: a list of all container info as dictionaries with the 
394                   keys "name", "count", and "bytes" 
395          @param limit: number of results to return, up to 10,000 
396          @type limit: int 
397          @param marker: return only results whose name is greater than "marker" 
398          @type marker: str 
399          """ 
400          if limit: 
401              parms['limit'] = limit 
402          if marker: 
403              parms['marker'] = marker 
404          parms['format'] = 'json' 
405          response = self.make_request('GET', [''], parms=parms) 
406          if (response.status < 200) or (response.status > 299): 
407              buff = response.read() 
408              raise ResponseError(response.status, response.reason) 
409          return json_loads(response.read()) 
 410   
412          """ 
413          Returns a list of Containers. 
414   
415          >>> connection.list_containers() 
416          ['new_container', 
417           'old_container', 
418           'container1', 
419           'container2', 
420           'container3', 
421           'test'] 
422   
423          @rtype: list(str) 
424          @return: a list of all containers names as strings 
425          @param limit: number of results to return, up to 10,000 
426          @type limit: int 
427          @param marker: return only results whose name is greater than "marker" 
428          @type marker: str 
429          """ 
430          if limit: 
431              parms['limit'] = limit 
432          if marker: 
433              parms['marker'] = marker 
434          response = self.make_request('GET', [''], parms=parms) 
435          if (response.status < 200) or (response.status > 299): 
436              buff = response.read() 
437              raise ResponseError(response.status, response.reason) 
438          return response.read().splitlines() 
 439   
441          """ 
442          Container objects can be grabbed from a connection using index 
443          syntax. 
444   
445          >>> container = conn['old_container'] 
446          >>> container.size_used 
447          23074 
448   
449          @rtype: L{Container} 
450          @return: an object representing the container 
451          """ 
452          return self.get_container(key) 
 453   
454   
456      """ 
457      A thread-safe connection pool object. 
458   
459      This component isn't required when using the cloudfiles library, but it may 
460      be useful when building threaded applications. 
461      """ 
462   
463 -    def __init__(self, username=None, api_key=None, **kwargs): 
 464          poolsize = kwargs.pop('poolsize', 10) 
465          self.connargs = {'username': username, 'api_key': api_key} 
466          self.connargs.update(kwargs) 
467          Queue.__init__(self, poolsize) 
 468   
470          """ 
471          Return a cloudfiles connection object. 
472   
473          @rtype: L{Connection} 
474          @return: a cloudfiles connection object 
475          """ 
476          try: 
477              (create, connobj) = Queue.get(self, block=0) 
478          except Empty: 
479              connobj = Connection(**self.connargs) 
480          return connobj 
 481   
482 -    def put(self, connobj): 
 483          """ 
484          Place a cloudfiles connection object back into the pool. 
485   
486          @param connobj: a cloudfiles connection object 
487          @type connobj: L{Connection} 
488          """ 
489          try: 
490              Queue.put(self, (time(), connobj), block=0) 
491          except Full: 
492              del connobj 
  493   
494