1
2
3
4
5 raise ImportError('This module is deprecated. Use gmPG2.py.')
6
7
8
9 """Broker for PostgreSQL distributed backend connections.
10
11 @copyright: author
12
13 TODO: iterator/generator batch fetching:
14 - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad
15 - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4"
16
17 winner:
18 def resultset_functional_batchgenerator(cursor, size=100):
19 for results in iter(lambda: cursor.fetchmany(size), []):
20 for rec in results:
21 yield rec
22 """
23
24
25 __version__ = "$Revision: 1.90 $"
26 __author__ = "H.Herb <hherb@gnumed.net>, I.Haywood <i.haywood@ugrad.unimelb.edu.au>, K.Hilbert <Karsten.Hilbert@gmx.net>"
27 __license__ = 'GPL (details at http://www.gnu.org)'
28
29 print "gmPG phased out, please replace with gmPG2"
30
31 import sys
32 sys.exit
33
34 _query_logging_verbosity = 1
35
36
37 assert(float(dbapi.apilevel) >= 2.0)
38 assert(dbapi.threadsafety > 0)
39 assert(dbapi.paramstyle == 'pyformat')
40
41 _listener_api = None
42
43
44 _default_client_encoding = {'wire': None, 'string': None}
45
46
47
48 if time.daylight:
49 tz = time.altzone
50 else:
51 tz = time.timezone
52
53
54 _default_client_timezone = "%+.1f" % (-tz / 3600.0)
55
56 _serialize_failure = "serialize access due to concurrent update"
57
58
59
60
61 QTablePrimaryKeyIndex = """
62 SELECT
63 indkey
64 FROM
65 pg_index
66 WHERE
67 indrelid =
68 (SELECT oid FROM pg_class WHERE relname = '%s');
69 """
70
71 query_pkey_name = """
72 SELECT
73 pga.attname
74 FROM
75 (pg_attribute pga inner join pg_index pgi on (pga.attrelid=pgi.indrelid))
76 WHERE
77 pga.attnum=pgi.indkey[0]
78 and
79 pgi.indisprimary is true
80 and
81 pga.attrelid=(SELECT oid FROM pg_class WHERE relname = %s)"""
82
83 query_fkey_names = """
84 select tgargs from pg_trigger where
85 tgname like 'RI%%'
86 and
87 tgrelid = (
88 select oid from pg_class where relname=%s
89 )
90 """
91
92
93 query_table_col_defs = """select
94 cols.column_name,
95 cols.udt_name
96 from
97 information_schema.columns cols
98 where
99 cols.table_schema = %s
100 and
101 cols.table_name = %s
102 order by
103 cols.ordinal_position"""
104
105 query_table_attributes = """select
106 cols.column_name
107 from
108 information_schema.columns cols
109 where
110 cols.table_schema = %s
111 and
112 cols.table_name = %s
113 order by
114 cols.ordinal_position"""
115
116 query_child_tables = """
117 select
118 pgn.nspname as namespace,
119 pgc.relname as table
120 from
121 pg_namespace pgn,
122 pg_class pgc
123 where
124 pgc.relnamespace = pgn.oid
125 and
126 pgc.oid in (
127 select inhrelid from pg_inherits where inhparent = (
128 select oid from pg_class where
129 relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and
130 relname = %(table)s
131 )
132 )"""
133
134
135
136 last_ro_cursor_desc = None
137
138
140 "maintains a static dictionary of available database connections"
141
142
143 __ro_conns = {}
144
145 __service2db_map = {}
146
147 __conn_use_count = {}
148
149 __is_connected = None
150
151 __listeners = {}
152
153 __login = None
154
155 - def __init__(self, login=None, encoding=None):
156 """parameter login is of type gmLoginInfo.LoginInfo"""
157
158 if login is not None:
159 self.__disconnect()
160 if ConnectionPool.__is_connected is None:
161
162 dbapi.fetchReturnsList = True
163 ConnectionPool.__is_connected = self.__setup_default_ro_conns(login=login, encoding=encoding)
164
167
168
169
170
171
172
173 - def GetConnection(self, service="default", readonly=1, encoding=None, extra_verbose=None):
203
216
219
220 - def get_connection_for_user(self, user=None, password=None, service="default", encoding=None, extra_verbose=None):
221 """Get a connection for a given user.
222
223 This will return a connection just as GetConnection() would
224 except that the user to be used for authentication can be
225 specified. All the other parameters are going to be the
226 same, IOW it will connect to the same server, port and database
227 as any other connection obtained through this broker.
228
229 You will have to specify the password, of course, if it
230 is needed for PostgreSQL authentication.
231
232 This will always return a read-write connection.
233 """
234 if user is None:
235 _log.Log(gmLog.lErr, 'user must be given')
236 raise ValueError, 'gmPG.py::%s.get_connection_for_user(): user name must be given' % self.__class__.__name__
237
238 logininfo = self.GetLoginInfoFor(service)
239 logininfo.SetUser(user=user)
240 logininfo.SetPassword(passwd=password)
241
242 _log.Log(gmLog.lData, "requesting RW connection to service [%s]" % service)
243 conn = self.__pgconnect(logininfo, readonly = 0, encoding = encoding)
244 if conn is None:
245 return None
246
247 if extra_verbose:
248 conn.conn.toggleShowQuery
249
250 return conn
251
252
253
254 - def Listen(self, service, signal, callback):
255 """Listen to 'signal' from backend in an asynchronous thread.
256
257 If 'signal' is received from database 'service', activate
258 the 'callback' function"""
259
260
261
262 if _listener_api is None:
263 if not _import_listener_engine():
264 _log.Log(gmLog.lErr, 'cannot load backend listener code')
265 return None
266
267
268 try:
269 backend = ConnectionPool.__service2db_map[service]
270 except KeyError:
271 backend = 0
272 _log.Log(gmLog.lData, "connecting notification [%s] from service [%s] (id %s) with callback %s" % (signal, service, backend, callback))
273
274
275 if backend not in ConnectionPool.__listeners.keys():
276 auth = self.GetLoginInfoFor(service)
277 listener = _listener_api.BackendListener(
278 service,
279 auth.GetDatabase(),
280 auth.GetUser(),
281 auth.GetPassword(),
282 auth.GetHost(),
283 int(auth.GetPort())
284 )
285 ConnectionPool.__listeners[backend] = listener
286
287 listener = ConnectionPool.__listeners[backend]
288 listener.register_callback(signal, callback)
289 return 1
290
291 - def Unlisten(self, service, signal, callback):
302
304 try:
305 backend = self.__service2db_map[service]
306 except KeyError:
307 _log.Log(gmLog.lWarn, 'cannot stop listener on backend')
308 return None
309 try:
310 ConnectionPool.__listeners[backend].stop_thread()
311 del ConnectionPool.__listeners[backend]
312 except:
313 _log.LogException('cannot stop listener on backend [%s]' % backend, sys.exc_info(), verbose = 0)
314 return None
315 return 1
316
325
326
327
329 """list all distributed services available on this system
330 (according to configuration database)"""
331 return ConnectionPool.__ro_conns.keys()
332
334 """return login information for a particular service"""
335 if login is None:
336 dblogin = ConnectionPool.__login
337 else:
338 dblogin = copy.deepcopy(login)
339
340 try:
341 srvc_id = ConnectionPool.__service2db_map[service]
342 except KeyError:
343 return dblogin
344
345 if srvc_id == 0:
346 return dblogin
347
348
349 cfg_db = ConnectionPool.__ro_conns['default']
350 cursor = cfg_db.cursor()
351 cmd = "select name, host, port from cfg.db where pk=%s"
352 if not run_query(cursor, None, cmd, srvc_id):
353 _log.Log(gmLog.lPanic, 'cannot get login info for service [%s] with id [%s] from config database' % (service, srvc_id))
354 _log.Log(gmLog.lPanic, 'make sure your service-to-database mappings are properly configured')
355 _log.Log(gmLog.lWarn, 'trying to make do with default login parameters')
356 return dblogin
357 auth_data = cursor.fetchone()
358 idx = get_col_indices(cursor)
359 cursor.close()
360
361 try:
362 dblogin.SetDatabase(string.strip(auth_data[idx['name']]))
363 except: pass
364 try:
365 dblogin.SetHost(string.strip(auth_data[idx['host']]))
366 except: pass
367 try:
368 dblogin.SetPort(auth_data[idx['port']])
369 except: pass
370
371 return dblogin
372
373
374
376 """Initialize connections to all servers."""
377 if login is None and ConnectionPool.__is_connected is None:
378 try:
379 login = request_login_params()
380 except:
381 _log.LogException("Exception: Cannot connect to databases without login information !", sys.exc_info(), verbose=1)
382 raise gmExceptions.ConnectionError("Can't connect to database without login information!")
383
384 _log.Log(gmLog.lData, login.GetInfoStr())
385 ConnectionPool.__login = login
386
387
388 cfg_db = self.__pgconnect(login, readonly=1, encoding=encoding)
389 if cfg_db is None:
390 raise gmExceptions.ConnectionError, _('Cannot connect to configuration database with:\n\n[%s]') % login.GetInfoStr()
391
392
393 ConnectionPool.__ro_conns['default'] = cfg_db
394 cursor = cfg_db.cursor()
395
396 cursor.execute("select version()")
397 _log.Log(gmLog.lInfo, 'service [default/config] running on [%s]' % cursor.fetchone()[0])
398
399 cmd = "select name from cfg.distributed_db"
400 if not run_query(cursor, None, cmd):
401 cursor.close()
402 raise gmExceptions.ConnectionError("cannot load service names from configuration database")
403 services = cursor.fetchall()
404 for service in services:
405 ConnectionPool.__service2db_map[service[0]] = 0
406
407
408
409 cmd = "select * from cfg.config where profile=%s"
410 if not run_query(cursor, None, cmd, login.GetProfile()):
411 cursor.close()
412 raise gmExceptions.ConnectionError("cannot load user profile [%s] from database" % login.GetProfile())
413 databases = cursor.fetchall()
414 dbidx = get_col_indices(cursor)
415
416
417 for db in databases:
418
419 cursor.execute("select name from cfg.distributed_db where pk=%d" % db[dbidx['ddb']])
420 service = string.strip(cursor.fetchone()[0])
421
422 _log.Log(gmLog.lData, "mapping service [%s] to DB ID [%s]" % (service, db[dbidx['db']]))
423 ConnectionPool.__service2db_map[service] = db[dbidx['db']]
424
425 ConnectionPool.__conn_use_count[service] = 0
426 dblogin = self.GetLoginInfoFor(service, login)
427
428 conn = self.__pgconnect(dblogin, readonly=1, encoding=encoding)
429 if conn is None:
430 raise gmExceptions.ConnectionError, _('Cannot connect to database with:\n\n[%s]') % login.GetInfoStr()
431 ConnectionPool.__ro_conns[service] = conn
432
433 cursor.execute("select version()")
434 _log.Log(gmLog.lInfo, 'service [%s] running on [%s]' % (service, cursor.fetchone()[0]))
435 cursor.close()
436 ConnectionPool.__is_connected = 1
437 return ConnectionPool.__is_connected
438
439 - def __pgconnect(self, login, readonly=1, encoding=None):
440 """Connect to a postgres backend as specified by login object.
441
442 - returns a connection object
443 - encoding works like this:
444 - encoding specified in the call to __pgconnect() overrides
445 - encoding set by a call to gmPG.set_default_encoding() overrides
446 - encoding taken from Python string encoding
447 - wire_encoding and string_encoding must essentially just be different
448 names for one and the same (IOW entirely compatible) encodings, such
449 as "win1250" and "cp1250"
450 """
451 dsn = ""
452 hostport = ""
453 dsn = login.GetDBAPI_DSN()
454 hostport = "0"
455
456 if encoding is None:
457 encoding = _default_client_encoding
458
459
460
461
462 string_encoding = encoding['string']
463 if string_encoding is None:
464 string_encoding = _default_client_encoding['string']
465 if string_encoding is None:
466
467 string_encoding = locale.getlocale()[1]
468 _log.Log(gmLog.lWarn, 'client encoding not specified, this may lead to data corruption in some cases')
469 _log.Log(gmLog.lWarn, 'therefore the string encoding currently set in the active locale is used: [%s]' % string_encoding)
470 _log.Log(gmLog.lWarn, 'for this to have any chance to work the application MUST have called locale.setlocale() before')
471 _log.Log(gmLog.lInfo, 'using string encoding [%s] to encode Unicode strings for transmission to the database' % string_encoding)
472
473
474
475
476 wire_encoding = encoding['wire']
477 if wire_encoding is None:
478 wire_encoding = _default_client_encoding['wire']
479 if wire_encoding is None:
480 wire_encoding = string_encoding
481 if wire_encoding is None:
482 raise ValueError, '<wire_encoding> cannot be None'
483
484 try:
485
486 conn = dbapi.connect(dsn=dsn, client_encoding=(string_encoding, 'strict'), unicode_results=1)
487 except StandardError:
488 _log.LogException("database connection failed: DSN = [%s], host:port = [%s]" % (dsn, hostport), sys.exc_info(), verbose = 1)
489 return None
490
491
492 curs = conn.cursor()
493
494
495 cmd = "set client_encoding to '%s'" % wire_encoding
496 try:
497 curs.execute(cmd)
498 except:
499 curs.close()
500 conn.close()
501 _log.Log(gmLog.lErr, 'query [%s]' % cmd)
502 _log.LogException (
503 'cannot set string-on-the-wire client_encoding on connection to [%s], this would likely lead to data corruption' % wire_encoding,
504 sys.exc_info(),
505 verbose = _query_logging_verbosity
506 )
507 raise
508 _log.Log(gmLog.lData, 'string-on-the-wire client_encoding set to [%s]' % wire_encoding)
509
510
511
512 cmd = "set time zone '%s'" % _default_client_timezone
513 if not run_query(curs, None, cmd):
514 _log.Log(gmLog.lErr, 'cannot set client time zone to [%s]' % _default_client_timezone)
515 _log.Log(gmLog.lWarn, 'not setting this will lead to incorrect dates/times')
516 else:
517 _log.Log (gmLog.lData, 'time zone set to [%s]' % _default_client_timezone)
518
519
520
521 cmd = "set datestyle to 'ISO'"
522 if not run_query(curs, None, cmd):
523 _log.Log(gmLog.lErr, 'cannot set client date style to ISO')
524 _log.Log(gmLog.lWarn, 'you better use other means to make your server delivers valid ISO timestamps with time zone')
525
526
527 if readonly:
528 isolation_level = 'READ COMMITTED'
529 else:
530 isolation_level = 'SERIALIZABLE'
531 cmd = 'set session characteristics as transaction isolation level %s' % isolation_level
532 if not run_query(curs, None, cmd):
533 curs.close()
534 conn.close()
535 _log.Log(gmLog.lErr, 'cannot set connection characteristics to [%s]' % isolation_level)
536 return None
537
538
539 if readonly:
540 access_mode = 'READ ONLY'
541 else:
542 access_mode = 'READ WRITE'
543 _log.Log(gmLog.lData, "setting session to [%s] for %s@%s:%s" % (access_mode, login.GetUser(), login.GetHost(), login.GetDatabase()))
544 cmd = 'set session characteristics as transaction %s' % access_mode
545 if not run_query(curs, 0, cmd):
546 _log.Log(gmLog.lErr, 'cannot set connection characteristics to [%s]' % access_mode)
547 curs.close()
548 conn.close()
549 return None
550
551 conn.commit()
552 curs.close()
553 return conn
554
581
582
583
584
586 "returns the attribute names of the fetched rows in natural sequence as a list"
587 names=[]
588 for d in cursor.description:
589 names.append(d[0])
590 return names
591
592 -def run_query(aCursor=None, verbosity=None, aQuery=None, *args):
593
594 if aCursor is None:
595 _log.Log(gmLog.lErr, 'need cursor to run query')
596 return None
597 if aQuery is None:
598 _log.Log(gmLog.lErr, 'need query to run it')
599 return None
600 if verbosity is None:
601 verbosity = _query_logging_verbosity
602
603
604 try:
605 aCursor.execute(aQuery, *args)
606 except:
607 _log.LogException("query >>>%s<<< with args >>>%s<<< failed" % (aQuery, args), sys.exc_info(), verbose = verbosity)
608 return None
609
610
611 return 1
612
613 -def run_commit2(link_obj=None, queries=None, end_tx=False, max_tries=1, extra_verbose=False, get_col_idx = False):
614 """Convenience function for running a transaction
615 that is supposed to get committed.
616
617 <link_obj>
618 can be either:
619 - a cursor
620 - a connection
621 - a service name
622
623 <queries>
624 is a list of (query, [args]) tuples to be
625 executed as a single transaction, the last
626 query may usefully return rows (such as a
627 "select currval('some_sequence')" statement)
628
629 <end_tx>
630 - controls whether the transaction is finalized (eg.
631 committed/rolled back) or not, this allows the
632 call to run_commit2() to be part of a framing
633 transaction
634 - if <link_obj> is a service name the transaction is
635 always finalized regardless of what <end_tx> says
636 - if link_obj is a connection then <end_tx> will
637 default to False unless it is explicitly set to
638 True which is taken to mean "yes, you do have full
639 control over the transaction" in which case the
640 transaction is properly finalized
641
642 <max_tries>
643 - controls the number of times a transaction is retried
644 after a concurrency error
645 - note that *all* <queries> are rerun if a concurrency
646 error occurrs
647 - max_tries is honored if and only if link_obj is a service
648 name such that we have full control over the transaction
649
650 <get_col_idx>
651 - if true, the returned data will include a dictionary
652 mapping field names to column positions
653 - if false, the returned data returns an empty dict
654
655 method result:
656 - returns a tuple (status, data)
657 - <status>:
658 * True - if all queries succeeded (also if there were 0 queries)
659 * False - if *any* error occurred
660 - <data> if <status> is True:
661 * (None, {}) if last query did not return rows
662 * ("fetchall() result", <index>) if last query returned any rows
663 * for <index> see <get_col_idx>
664 - <data> if <status> is False:
665 * a tuple (error, message) where <error> can be:
666 * 1: unspecified error
667 * 2: concurrency error
668 * 3: constraint violation (non-primary key)
669 * 4: access violation
670 """
671
672 if queries is None:
673 return (False, (1, 'forgot to pass in queries'))
674 if len(queries) == 0:
675 return (True, 'no queries to execute')
676
677
678
679 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'):
680 return __commit2cursor(cursor=link_obj, queries=queries, extra_verbose=extra_verbose, get_col_idx=get_col_idx)
681
682 if (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')):
683 return __commit2conn(conn=link_obj, queries=queries, end_tx=end_tx, extra_verbose=extra_verbose, get_col_idx=get_col_idx)
684
685 return __commit2service(service=link_obj, queries=queries, max_tries=max_tries, extra_verbose=extra_verbose, get_col_idx=get_col_idx)
686
687 -def __commit2service(service=None, queries=None, max_tries=1, extra_verbose=False, get_col_idx=False):
688
689 try: int(max_tries)
690 except ValueEror: max_tries = 1
691 if max_tries > 4:
692 max_tries = 4
693 if max_tries < 1:
694 max_tries = 1
695
696 pool = ConnectionPool()
697 conn = pool.GetConnection(str(service), readonly = 0)
698 if conn is None:
699 msg = 'cannot connect to service [%s]'
700 _log.Log(gmLog.lErr, msg % service)
701 return (False, (1, _(msg) % service))
702 if extra_verbose:
703 conn.conn.toggleShowQuery
704 curs = conn.cursor()
705 for attempt in range(0, max_tries):
706 if extra_verbose:
707 _log.Log(gmLog.lData, 'attempt %s' % attempt)
708
709 for query, args in queries:
710 if extra_verbose:
711 t1 = time.time()
712 try:
713 curs.execute(query, *args)
714
715 except:
716 if extra_verbose:
717 duration = time.time() - t1
718 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration)
719 conn.rollback()
720 exc_info = sys.exc_info()
721 typ, val, tb = exc_info
722 if str(val).find(_serialize_failure) > 0:
723 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update')
724 if attempt < max_tries:
725
726 time.sleep(0.1)
727 continue
728 curs.close()
729 conn.close()
730 return (False, (2, 'l'))
731
732 _log.Log(gmLog.lErr, 'query: %s' % query[:2048])
733 try:
734 _log.Log(gmLog.lErr, 'argument: %s' % str(args)[:2048])
735 except MemoryError:
736 pass
737 _log.LogException("query failed on link [%s]" % service, exc_info)
738 if extra_verbose:
739 __log_PG_settings(curs)
740 curs.close()
741 conn.close()
742 tmp = str(val).replace('ERROR:', '')
743 tmp = tmp.replace('ExecAppend:', '')
744 tmp = tmp.strip()
745 return (False, (1, _('SQL: %s') % tmp))
746
747 if extra_verbose:
748 duration = time.time() - t1
749 _log.Log(gmLog.lData, 'query: %s' % query[:2048])
750 try:
751 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048])
752 except MemoryError:
753 pass
754 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % service)
755 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration))
756
757 break
758
759
760 data = None
761 idx = {}
762
763
764
765
766
767
768 try:
769 data = curs.fetchall()
770 except:
771 if extra_verbose:
772 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows')
773
774 if curs.description is not None:
775 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?')
776 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description))
777 conn.commit()
778 if get_col_idx:
779 idx = get_col_indices(curs)
780 curs.close()
781 conn.close()
782 return (True, (data, idx))
783
784 -def __commit2conn(conn=None, queries=None, end_tx=False, extra_verbose=False, get_col_idx=False):
785 if extra_verbose:
786 conn.conn.toggleShowQuery
787
788
789 curs = conn.cursor()
790
791
792 for query, args in queries:
793 if extra_verbose:
794 t1 = time.time()
795 try:
796 curs.execute(query, *args)
797 except:
798 if extra_verbose:
799 duration = time.time() - t1
800 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration)
801 conn.rollback()
802 exc_info = sys.exc_info()
803 typ, val, tb = exc_info
804 if str(val).find(_serialize_failure) > 0:
805 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update')
806 curs.close()
807 if extra_verbose:
808 conn.conn.toggleShowQuery
809 return (False, (2, 'l'))
810
811 _log.Log(gmLog.lErr, 'query: %s' % query[:2048])
812 try:
813 _log.Log(gmLog.lErr, 'args : %s' % str(args)[:2048])
814 except MemoryError:
815 pass
816 _log.LogException("query failed on link [%s]" % conn, exc_info)
817 if extra_verbose:
818 __log_PG_settings(curs)
819 curs.close()
820 tmp = str(val).replace('ERROR:', '')
821 tmp = tmp.replace('ExecAppend:', '')
822 tmp = tmp.strip()
823 if extra_verbose:
824 conn.conn.toggleShowQuery
825 return (False, (1, _('SQL: %s') % tmp))
826
827 if extra_verbose:
828 duration = time.time() - t1
829 _log.Log(gmLog.lData, 'query: %s' % query[:2048])
830 try:
831 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048])
832 except MemoryError:
833 pass
834 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % conn)
835 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration))
836
837 if extra_verbose:
838 conn.conn.toggleShowQuery
839
840 data = None
841 idx = {}
842
843
844
845
846
847
848 try:
849 data = curs.fetchall()
850 except:
851 if extra_verbose:
852 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows')
853
854 if curs.description is not None:
855 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?')
856 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description))
857 if end_tx:
858 conn.commit()
859 if get_col_idx:
860 idx = get_col_indices(curs)
861 curs.close()
862 return (True, (data, idx))
863
864 -def __commit2cursor(cursor=None, queries=None, extra_verbose=False, get_col_idx=False):
865
866 for query, args in queries:
867 if extra_verbose:
868 t1 = time.time()
869 try:
870 curs.execute(query, *args)
871 except:
872 if extra_verbose:
873 duration = time.time() - t1
874 _log.Log(gmLog.lData, 'query took %3.3f seconds' % duration)
875 exc_info = sys.exc_info()
876 typ, val, tb = exc_info
877 if str(val).find(_serialize_failure) > 0:
878 _log.Log(gmLog.lData, 'concurrency conflict detected, cannot serialize access due to concurrent update')
879 return (False, (2, 'l'))
880
881 _log.Log(gmLog.lErr, 'query: %s' % query[:2048])
882 try:
883 _log.Log(gmLog.lErr, 'args : %s' % str(args)[:2048])
884 except MemoryError:
885 pass
886 _log.LogException("query failed on link [%s]" % cursor, exc_info)
887 if extra_verbose:
888 __log_PG_settings(curs)
889 tmp = str(val).replace('ERROR:', '')
890 tmp = tmp.replace('ExecAppend:', '')
891 tmp = tmp.strip()
892 return (False, (1, _('SQL: %s') % tmp))
893
894 if extra_verbose:
895 duration = time.time() - t1
896 _log.Log(gmLog.lData, 'query: %s' % query[:2048])
897 try:
898 _log.Log(gmLog.lData, 'args : %s' % str(args)[:2048])
899 except MemoryError:
900 pass
901 _log.Log(gmLog.lData, 'query succeeded on link [%s]' % cursor)
902 _log.Log(gmLog.lData, '%s rows affected/returned in %3.3f seconds' % (curs.rowcount, duration))
903
904
905 data = None
906 idx = {}
907
908
909
910
911
912
913 try:
914 data = curs.fetchall()
915 except:
916 if extra_verbose:
917 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows')
918
919 if curs.description is not None:
920 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?')
921 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description))
922 if get_col_idx:
923 idx = get_col_indices(curs)
924 return (True, (data, idx))
925
926 -def run_commit(link_obj = None, queries = None, return_err_msg = None):
927 """Convenience function for running a transaction
928 that is supposed to get committed.
929
930 - link_obj can be
931 - a cursor: rollback/commit must be done by the caller
932 - a connection: rollback/commit is handled
933 - a service name: rollback/commit is handled
934
935 - queries is a list of (query, [args]) tuples
936 - executed as a single transaction
937
938 - returns:
939 - a tuple (<value>, error) if return_err_msg is True
940 - a scalar <value> if return_err_msg is False
941
942 - <value> will be
943 - None: if any query failed
944 - 1: if all queries succeeded (also 0 queries)
945 - data: if the last query returned rows
946 """
947 print "DEPRECATION WARNING: gmPG.run_commit() is deprecated, use run_commit2() instead"
948
949
950 if link_obj is None:
951 raise TypeError, 'gmPG.run_commit(): link_obj must be of type service name, connection or cursor'
952 if queries is None:
953 raise TypeError, 'gmPG.run_commit(): forgot to pass in queries'
954 if len(queries) == 0:
955 _log.Log(gmLog.lWarn, 'no queries to execute ?!?')
956 if return_err_msg:
957 return (1, 'no queries to execute ?!?')
958 return 1
959
960 close_cursor = noop
961 close_conn = noop
962 commit = noop
963 rollback = noop
964
965 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'):
966 curs = link_obj
967
968 elif (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')):
969 curs = link_obj.cursor()
970 close_cursor = curs.close
971 conn = link_obj
972 commit = link_obj.commit
973 rollback = link_obj.rollback
974
975 else:
976 pool = ConnectionPool()
977 conn = pool.GetConnection(link_obj, readonly = 0)
978 if conn is None:
979 _log.Log(gmLog.lErr, 'cannot connect to service [%s]' % link_obj)
980 if return_err_msg:
981 return (None, _('cannot connect to service [%s]') % link_obj)
982 return None
983 curs = conn.cursor()
984 close_cursor = curs.close
985 close_conn = conn.close
986 commit = conn.commit
987 rollback = conn.rollback
988
989 for query, args in queries:
990
991 try:
992 curs.execute (query, *args)
993 except:
994 rollback()
995 exc_info = sys.exc_info()
996 _log.LogException ("RW query >>>%s<<< with args >>>%s<<< failed on link [%s]" % (query[:1024], str(args)[:1024], link_obj), exc_info, verbose = _query_logging_verbosity)
997 __log_PG_settings(curs)
998 close_cursor()
999 close_conn()
1000 if return_err_msg:
1001 typ, val, tb = exc_info
1002 tmp = string.replace(str(val), 'ERROR:', '')
1003 tmp = string.replace(tmp, 'ExecAppend:', '')
1004 tmp = string.strip(tmp)
1005 return (None, 'SQL: %s' % tmp)
1006 return None
1007
1008
1009 if _query_logging_verbosity == 1:
1010 _log.Log(gmLog.lData, '%s rows affected by >>>%s<<<' % (curs.rowcount, query))
1011
1012 data = None
1013
1014
1015
1016
1017
1018 try:
1019 data = curs.fetchall()
1020 if _query_logging_verbosity == 1:
1021 _log.Log(gmLog.lData, 'last query returned %s rows' % curs.rowcount)
1022 except:
1023 if _query_logging_verbosity == 1:
1024 _log.Log(gmLog.lData, 'fetchall(): last query did not return rows')
1025
1026 if curs.description is not None:
1027 if curs.rowcount > 0:
1028 _log.Log(gmLog.lData, 'there seem to be rows but fetchall() failed -- DB API violation ?')
1029 _log.Log(gmLog.lData, 'rowcount: %s, description: %s' % (curs.rowcount, curs.description))
1030
1031
1032 commit()
1033 close_cursor()
1034 close_conn()
1035
1036 if data is None: status = 1
1037 else: status = data
1038 if return_err_msg: return (status, '')
1039 return status
1040
1041 -def run_ro_query(link_obj = None, aQuery = None, get_col_idx = False, *args):
1042 """Runs a read-only query.
1043
1044 - link_obj can be a service name, connection or cursor object
1045
1046 - return status:
1047 - return data if get_col_idx is False
1048 - return (data, idx) if get_col_idx is True
1049
1050 - if query fails: data is None
1051 - if query is not a row-returning SQL statement: data is None
1052
1053 - data is a list of tuples [(w,x,y,z), (a,b,c,d), ...] where each tuple is a table row
1054 - idx is a map of column name to their position in the row tuples
1055 e.g. { 'name': 3, 'id':0, 'job_description': 2, 'location':1 }
1056
1057 usage: e.g. data[0][idx['name']] would return z from [(w,x,y,z ),(a,b,c,d)]
1058 """
1059
1060 if link_obj is None:
1061 raise TypeError, 'gmPG.run_ro_query(): link_obj must be of type service name, connection or cursor'
1062 if aQuery is None:
1063 raise TypeError, 'gmPG.run_ro_query(): forgot to pass in aQuery'
1064
1065 close_cursor = noop
1066 close_conn = noop
1067
1068 if hasattr(link_obj, 'fetchone') and hasattr(link_obj, 'description'):
1069 curs = link_obj
1070
1071 elif (hasattr(link_obj, 'commit') and hasattr(link_obj, 'cursor')):
1072 curs = link_obj.cursor()
1073 close_cursor = curs.close
1074
1075 else:
1076 pool = ConnectionPool()
1077 conn = pool.GetConnection(link_obj, readonly = 1)
1078 if conn is None:
1079 _log.Log(gmLog.lErr, 'cannot get connection to service [%s]' % link_obj)
1080 if not get_col_idx:
1081 return None
1082 else:
1083 return None, None
1084 curs = conn.cursor()
1085 close_cursor = curs.close
1086 close_conn = pool.ReleaseConnection
1087
1088
1089 try:
1090 curs.execute(aQuery, *args)
1091 global last_ro_cursor_desc
1092 last_ro_cursor_desc = curs.description
1093 except:
1094 _log.LogException("query >>>%s<<< with args >>>%s<<< failed on link [%s]" % (aQuery[:250], str(args)[:250], link_obj), sys.exc_info(), verbose = _query_logging_verbosity)
1095 __log_PG_settings(curs)
1096 close_cursor()
1097 close_conn(link_obj)
1098 if not get_col_idx:
1099 return None
1100 else:
1101 return None, None
1102
1103
1104
1105 if curs.description is None:
1106 data = None
1107 _log.Log(gmLog.lErr, 'query did not return rows')
1108 else:
1109 try:
1110 data = curs.fetchall()
1111 except:
1112 _log.LogException('cursor.fetchall() failed on link [%s]' % link_obj, sys.exc_info(), verbose = _query_logging_verbosity)
1113 close_cursor()
1114 close_conn(link_obj)
1115 if not get_col_idx:
1116 return None
1117 else:
1118 return None, None
1119
1120
1121 close_conn(link_obj)
1122 if get_col_idx:
1123 col_idx = get_col_indices(curs)
1124 close_cursor()
1125 return data, col_idx
1126 else:
1127 close_cursor()
1128 return data
1129
1130
1132
1133 if aCursor is None:
1134 _log.Log(gmLog.lErr, 'need cursor to get column indices')
1135 return None
1136 if aCursor.description is None:
1137 _log.Log(gmLog.lErr, 'no result description available: cursor unused or last query did not select rows')
1138 return None
1139 col_indices = {}
1140 col_index = 0
1141 for col_desc in aCursor.description:
1142 col_indices[col_desc[0]] = col_index
1143 col_index += 1
1144 return col_indices
1145
1146
1147
1149
1150 if aCursor is None:
1151 _log.Log(gmLog.lErr, 'need cursor to determine primary key')
1152 return None
1153 if aTable is None:
1154 _log.Log(gmLog.lErr, 'need table name for which to determine primary key')
1155
1156 if not run_query(aCursor, None, query_pkey_name, aTable):
1157 _log.Log(gmLog.lErr, 'cannot determine primary key')
1158 return -1
1159 result = aCursor.fetchone()
1160 if result is None:
1161 return None
1162 return result[0]
1163
1165 """Returns a dictionary of referenced foreign keys.
1166
1167 key = column name of this table
1168 value = (referenced table name, referenced column name) tuple
1169 """
1170 manage_connection = 0
1171 close_cursor = 1
1172
1173 if hasattr(source, 'fetchone') and hasattr(source, 'description'):
1174 close_cursor = 0
1175 curs = source
1176
1177 elif (hasattr(source, 'commit') and hasattr(source, 'cursor')):
1178 curs = source.cursor()
1179
1180 else:
1181 manage_connection = 1
1182 pool = ConnectionPool()
1183 conn = pool.GetConnection(source)
1184 if conn is None:
1185 _log.Log(gmLog.lErr, 'cannot get fkey names on table [%s] from source [%s]' % (table, source))
1186 return None
1187 curs = conn.cursor()
1188
1189 if not run_query(curs, None, query_fkey_names, table):
1190 if close_cursor:
1191 curs.close()
1192 if manage_connection:
1193 pool.ReleaseConnection(source)
1194 _log.Log(gmLog.lErr, 'cannot get foreign keys on table [%s] from source [%s]' % (table, source))
1195 return None
1196
1197 fks = curs.fetchall()
1198 if close_cursor:
1199 curs.close()
1200 if manage_connection:
1201 pool.ReleaseConnection(source)
1202
1203 references = {}
1204 for fk in fks:
1205 fkname, src_table, target_table, tmp, src_col, target_col, tmp = string.split(fk[0], '\x00')
1206 references[src_col] = (target_table, target_col)
1207
1208 return references
1209
1210 -def add_housekeeping_todo(
1211 reporter='$RCSfile: gmPG.py,v $ $Revision: 1.90 $',
1212 receiver='DEFAULT',
1213 problem='lazy programmer',
1214 solution='lazy programmer',
1215 context='lazy programmer',
1216 category='lazy programmer'
1217 ):
1218 queries = []
1219 cmd = "insert into housekeeping_todo (reported_by, reported_to, problem, solution, context, category) values (%s, %s, %s, %s, %s, %s)"
1220 queries.append((cmd, [reporter, receiver, problem, solution, context, category]))
1221 cmd = "select currval('housekeeping_todo_pk_seq')"
1222 queries.append((cmd, []))
1223 result, err = run_commit('historica', queries, 1)
1224 if result is None:
1225 _log.Log(gmLog.lErr, err)
1226 return (None, err)
1227 return (1, result[0][0])
1228
1230
1231 def myCallback(**kwds):
1232 sys.stdout.flush()
1233 print "\n=== myCallback: got called ==="
1234 print kwds
1235
1236
1237 dbpool = ConnectionPool()
1238 roconn = dbpool.GetConnection('default', extra_verbose=1)
1239 rocurs = roconn.cursor()
1240
1241
1242 print "PostgreSQL backend listener debug shell"
1243 while 1:
1244 print "---------------------------------------"
1245 typed = raw_input("=> ")
1246 args = typed.split(' ')
1247
1248 if len(args) == 0:
1249 continue
1250
1251 if args[0] in ('help', '?'):
1252 print "known commands"
1253 print "--------------"
1254 print "'listen' - start listening to a signal"
1255 print "'ignore' - stop listening to a signal"
1256 print "'send' - send a signal"
1257 print "'quit', 'exit', 'done' - well, chicken out"
1258 continue
1259
1260 if args[0] in ('quit', 'exit', 'done'):
1261 break
1262
1263 if args[0] in ("listen", "ignore", "send"):
1264 typed = raw_input("signal name: ")
1265 sig_names = typed.split(' ')
1266
1267 if len(sig_names) == 0:
1268 continue
1269 if args[0] == "listen":
1270 dbpool.Listen('default', sig_names[0], myCallback)
1271 if args[0] == "ignore":
1272 dbpool.Unlisten('default', sig_names[0], myCallback)
1273 if args[0] == "send":
1274 cmd = 'NOTIFY "%s"' % sig_names[0]
1275 print "... running >>>%s<<<" % (cmd)
1276 if not run_query(rocurs, None, cmd):
1277 print "... error sending [%s]" % cmd
1278 roconn.commit()
1279 continue
1280 print 'unknown command [%s]' % typed
1281
1282
1283 print "please wait a second or two for threads to sync and die"
1284 dbpool.StopListener('default')
1285 rocurs.close()
1286 roconn.close()
1287 dbpool.ReleaseConnection('default')
1288
1289
1290
1291 if __name__ == "__main__":
1292 _log.Log(gmLog.lData, 'DBMS "%s" via DB-API module "%s": API level %s, thread safety %s, parameter style "%s"' % ('PostgreSQL', dbapi, dbapi.apilevel, dbapi.threadsafety, dbapi.paramstyle))
1293
1294 print "Do you want to test the backend notification code ?"
1295 yes_no = raw_input('y/n: ')
1296 if yes_no == 'y':
1297 __run_notifications_debugger()
1298 sys.exit()
1299
1300 dbpool = ConnectionPool()
1301
1302 print "\n\nServices available on this system:"
1303 print '-----------------------------------------'
1304 for service in dbpool.GetAvailableServices():
1305 print service
1306 dummy = dbpool.GetConnection(service)
1307 print "\n.......................................\n"
1308
1309
1310 db = dbpool.GetConnection('config')
1311 print "\n\nPossible services on any gnumed system:"
1312 print '-----------------------------------------'
1313 cursor = db.cursor()
1314 cursor.execute("select name from cfg.distributed_db")
1315 for service in cursor.fetchall():
1316 print service[0]
1317
1318 print "\nTesting convenience funtions:\n============================\n"
1319
1320 print "\nResult as dictionary\n==================\n"
1321 cur = db.cursor()
1322 cursor.execute("select * from cfg.db")
1323 d = dictResult(cursor)
1324 print d
1325 print "\nResult attributes\n==================\n"
1326 n = fieldNames(cursor)
1327
1329 print "[Backend notification received!]"
1330
1331 print "\n-------------------------------------"
1332 print "Testing asynchronous notification for approx. 20 seconds"
1333 print "start psql in another window connect to gnumed"
1334 print "and type 'notify test'; if everything works,"
1335 print "a message [Backend notification received!] should appear\n"
1336 dbpool.Listen('default', 'test', TestCallback)
1337 time.sleep(20)
1338 dbpool.StopListener('default')
1339 print "Requesting write access connection:"
1340 con = dbpool.GetConnection('default', readonly=0)
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914