| [180] | 1 | import urllib |
|---|
| 2 | import urllib2 |
|---|
| 3 | import time |
|---|
| 4 | import datetime |
|---|
| 5 | |
|---|
| 6 | class ECMWFDataServer: |
|---|
| 7 | def __init__(self,portal,token,email): |
|---|
| 8 | self.version = '0.3' |
|---|
| 9 | self.portal = portal |
|---|
| 10 | self.token = token |
|---|
| 11 | self.email = email |
|---|
| 12 | |
|---|
| 13 | def _call(self,action,args): |
|---|
| 14 | |
|---|
| 15 | params = {'_token' : self.token, |
|---|
| 16 | '_email' : self.email, |
|---|
| 17 | '_action' : action, |
|---|
| 18 | '_version' : self.version} |
|---|
| 19 | params.update(args) |
|---|
| 20 | |
|---|
| 21 | data = urllib.urlencode(params) |
|---|
| 22 | req = urllib2.Request(self.portal, data) |
|---|
| 23 | response = urllib2.urlopen(req) |
|---|
| 24 | |
|---|
| 25 | json = response.read(); |
|---|
| 26 | |
|---|
| 27 | undef = None; |
|---|
| 28 | json = eval(json) |
|---|
| 29 | |
|---|
| 30 | if json != None: |
|---|
| 31 | if 'error' in json: |
|---|
| 32 | raise RuntimeError(json['error']) |
|---|
| 33 | if 'message' in json: |
|---|
| 34 | self.put(json['message']) |
|---|
| 35 | |
|---|
| 36 | return json |
|---|
| 37 | |
|---|
| 38 | def put(self,*args): |
|---|
| 39 | print datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), |
|---|
| 40 | for a in args: |
|---|
| 41 | print a, |
|---|
| 42 | print |
|---|
| 43 | |
|---|
| 44 | |
|---|
| 45 | def retrieve(self,args): |
|---|
| 46 | self.put("ECMWF data server batch tool version",self.version); |
|---|
| 47 | user = self._call("user_info",{}); |
|---|
| 48 | self.put("Welcome to",user['name'], "from" , user['organisation']); |
|---|
| 49 | |
|---|
| 50 | r = self._call('retrieve',args) |
|---|
| 51 | rid = r['request'] |
|---|
| 52 | |
|---|
| 53 | last = '' |
|---|
| 54 | sleep = 0 |
|---|
| 55 | while r['status'] != 'complete' and r['status'] != 'aborted': |
|---|
| 56 | text = r['status'] + '.' |
|---|
| 57 | if 'info' in r and r['info'] != None: |
|---|
| 58 | text = text + ' ' + r['info'] |
|---|
| 59 | |
|---|
| 60 | if text != last: |
|---|
| 61 | self.put("Request",text) |
|---|
| 62 | last = text |
|---|
| 63 | |
|---|
| 64 | time.sleep(sleep) |
|---|
| 65 | r = self._call('status',{'request':rid}) |
|---|
| 66 | if sleep < 60: |
|---|
| 67 | sleep = sleep + 1 |
|---|
| 68 | |
|---|
| 69 | if r['status'] != last: |
|---|
| 70 | self.put("Request",r['status']) |
|---|
| 71 | |
|---|
| 72 | if 'reason' in r: |
|---|
| 73 | for m in r['reason']: |
|---|
| 74 | self.put(m) |
|---|
| 75 | |
|---|
| 76 | if 'result' in r: |
|---|
| 77 | size = long(r['size']) |
|---|
| 78 | self.put("Downloading",self._bytename(size)) |
|---|
| 79 | done = self._transfer(r['result'],args['target']) |
|---|
| 80 | self.put("Done") |
|---|
| 81 | if done != size: |
|---|
| 82 | raise RuntimeError("Size mismatch: " + str(done) + " and " + str(size)) |
|---|
| 83 | |
|---|
| 84 | self._call('delete',{'request':rid}) |
|---|
| 85 | |
|---|
| 86 | if r['status'] == 'aborted': |
|---|
| 87 | raise RuntimeError("Request aborted") |
|---|
| 88 | |
|---|
| 89 | def _transfer(self,url,path): |
|---|
| 90 | result = urllib.urlretrieve(url,path) |
|---|
| 91 | return long(result[1]['content-length']) |
|---|
| 92 | |
|---|
| 93 | def _bytename(self,size): |
|---|
| 94 | next = {'':'K','K':'M','M':'G','G':'T','T':'P'} |
|---|
| 95 | l = '' |
|---|
| 96 | size = size*1.0 |
|---|
| 97 | while 1024 < size: |
|---|
| 98 | l = next[l] |
|---|
| 99 | size = size / 1024 |
|---|
| 100 | return "%g %sbyte%s" % (size,l,'s') |
|---|
| 101 | |
|---|
