/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to effbot/org/http_manager.py

  • Committer: Martin Pool
  • Date: 2005-08-05 19:38:50 UTC
  • Revision ID: mbp@sourcefrog.net-20050805193850-d3d034f15388791a
- add some comments on merge from talking to aaron

Show diffs side-by-side

added added

removed removed

Lines of Context:
effbot/org/http_manager.py
 
1
# $Id: http_manager.py 270 2004-10-09 10:38:54Z fredrik $
 
2
# effnews http
 
3
#
 
4
# manage a set of http clients
 
5
#
 
6
# Copyright (c) 2001-2004 by Fredrik Lundh.  All rights reserved.
 
7
#
 
8
 
 
9
import asyncore, time
 
10
import http_client
 
11
 
 
12
class http_manager:
 
13
 
 
14
    max_connections = 8
 
15
    max_size = 1000000
 
16
    max_time = 60
 
17
 
 
18
    def __init__(self):
 
19
        self.queue = []
 
20
 
 
21
    def request(self, uri, consumer, extra_headers=None):
 
22
        self.queue.append((uri, consumer, extra_headers))
 
23
 
 
24
    def priority_request(self, uri, consumer, extra_headers=None):
 
25
        self.queue.insert(0, (uri, consumer, extra_headers))
 
26
 
 
27
    def purge(self):
 
28
        for channel in asyncore.socket_map.values():
 
29
            channel.close()
 
30
        del self.queue[:]
 
31
 
 
32
    def prioritize(self, priority_uri):
 
33
        i = 0
 
34
        for uri, consumer, extra_headers in self.queue:
 
35
            if uri == priority_uri:
 
36
                del self.queue[i]
 
37
                self.priority_request(uri, consumer, extra_headers)
 
38
                return
 
39
            i = i + 1
 
40
 
 
41
    def poll(self, timeout=0.1):
 
42
        # sanity checks
 
43
        now = time.time()
 
44
        for channel in asyncore.socket_map.values():
 
45
            if channel.bytes_in > self.max_size:
 
46
                channel.close() # too much data
 
47
                try:
 
48
                    channel.consumer.http(
 
49
                        0, channel, ("HTTPManager", "too much data", None)
 
50
                        )
 
51
                except:
 
52
                    pass
 
53
            if channel.timestamp and now - channel.timestamp > self.max_time:
 
54
                channel.close() # too slow
 
55
                try:
 
56
                    channel.consumer.http(
 
57
                        0, channel, ("HTTPManager", "timeout", None)
 
58
                        )
 
59
                except:
 
60
                    pass
 
61
        # activate up to max_connections channels
 
62
        while self.queue and len(asyncore.socket_map) < self.max_connections:
 
63
            http_client.do_request(*self.queue.pop(0))
 
64
        # keep the network running
 
65
        asyncore.poll(timeout)
 
66
        # return non-zero if we should keep on polling
 
67
        return len(self.queue) or len(asyncore.socket_map)