Partial patch for ITS#6157 deadlock.

Does not address abandon(Cancel/StartTLS), syncprov, syncrepl.
Don't know if the bconfig.c pathces are correct.

Index: slap.h
===================================================================
RCS file: /repo/OpenLDAP/pkg/ldap/servers/slapd/slap.h,v
retrieving revision 1.873
diff -u -2 -r1.873 slap.h
--- slap.h	5 Mar 2009 19:43:53 -0000	1.873
+++ slap.h	3 Jun 2009 12:37:07 -0000
@@ -2602,4 +2602,5 @@
 #define SLAP_CANCEL_ACK					0x02
 #define SLAP_CANCEL_DONE				0x03
+#define SLAP_CANCEL_INVALID				0x04 /* like 0 but prevents Cancel */
 
 	GroupAssertion *o_groups;
Index: proto-slap.h
===================================================================
RCS file: /repo/OpenLDAP/pkg/ldap/servers/slapd/proto-slap.h,v
retrieving revision 1.784
diff -u -2 -r1.784 proto-slap.h
--- proto-slap.h	2 Jun 2009 21:17:59 -0000	1.784
+++ proto-slap.h	3 Jun 2009 12:37:06 -0000
@@ -1044,4 +1044,5 @@
  */
 LDAP_SLAPD_F ( SLAP_EXTOP_MAIN_FN ) cancel_extop;
+LDAP_SLAPD_F ( int ) prevent_cancel LDAP_P(( Operation *op, int need_lock ));
 
 /*
Index: cancel.c
===================================================================
RCS file: /repo/OpenLDAP/pkg/ldap/servers/slapd/cancel.c,v
retrieving revision 1.32
diff -u -2 -r1.32 cancel.c
--- cancel.c	22 May 2009 16:10:39 -0000	1.32
+++ cancel.c	3 Jun 2009 12:40:56 -0000
@@ -67,4 +67,10 @@
 	ldap_pvt_thread_mutex_lock( &op->o_conn->c_mutex );
 
+	if ( prevent_cancel( op, 0 ) != LDAP_SUCCESS ) {
+		rc = LDAP_OPERATIONS_ERROR;
+		rs->sr_text = "tried to abandon or cancel this Cancel";
+		goto out;
+	}
+
 	LDAP_STAILQ_FOREACH( o, &op->o_conn->c_pending_ops, o_next ) {
 		if ( o->o_msgid == opid ) {
@@ -89,13 +95,35 @@
 	 	rc = LDAP_NO_SUCH_OPERATION;
 		rs->sr_text = "message ID not found";
-	} else if ( o->o_cancel != SLAP_CANCEL_NONE ) {
-		rc = LDAP_PROTOCOL_ERROR;
-		rs->sr_text = "message ID already being cancelled";
+
+	} else if ( o->o_tag == LDAP_REQ_BIND
+			|| o->o_tag == LDAP_REQ_UNBIND
+			|| o->o_tag == LDAP_REQ_ABANDON ) {
+		rc = LDAP_CANNOT_CANCEL;
+
 	} else {
-		rc = LDAP_SUCCESS;
-		o->o_cancel = SLAP_CANCEL_REQ;
-		o->o_abandon = 1;
+		switch ( o->o_cancel ) {
+		case SLAP_CANCEL_NONE:
+			if ( !o->o_abandon ) {
+				rc = LDAP_SUCCESS;
+				o->o_cancel = SLAP_CANCEL_REQ;
+				o->o_abandon = 1;
+				break;
+			}
+			/* Already abandoned, or ITS#6138: o_abandon="suppress response" */
+			rc = LDAP_TOO_LATE;
+			break;
+
+		case SLAP_CANCEL_INVALID:
+			rc = LDAP_CANNOT_CANCEL;
+			break;
+
+		default:
+			rc = LDAP_OPERATIONS_ERROR;
+			rs->sr_text = "message ID already being cancelled";
+			break;
+		}
 	}
 
+ out:
 	ldap_pvt_thread_mutex_unlock( &op->o_conn->c_mutex );
 
@@ -110,14 +138,13 @@
 		}
 
-		while ( o->o_cancel == SLAP_CANCEL_REQ ) {
+		while ( (rc = o->o_cancel) == SLAP_CANCEL_REQ ) {
 			ldap_pvt_thread_yield();
 		}
 
-		if ( o->o_cancel == SLAP_CANCEL_ACK ) {
+		if ( rc == SLAP_CANCEL_ACK ) {
 			rc = LDAP_SUCCESS;
-		} else {
-			rc = o->o_cancel;
 		}
 
+		assert( rc != SLAP_CANCEL_INVALID );
 		o->o_cancel = SLAP_CANCEL_DONE;
 	}
@@ -125,2 +152,35 @@
 	return rc;
 }
+
+/*
+ * Return SLAPD_ABANDON if operation is abandoned or cancelled,
+ * otherwise return LDAP_SUCCESS after rejecting future Cancels.
+ * Needs to lock op->o_conn->c_mutex if not already locked.
+ */
+int
+prevent_cancel( Operation *op, int need_lock )
+{
+	int rc = LDAP_SUCCESS, cancel;
+
+	if ( need_lock ) {
+		if ( op->o_conn == NULL ) { /* dunno if this is needed */
+			need_lock = 0;
+		} else {
+			ldap_pvt_thread_mutex_lock( &op->o_conn->c_mutex );
+		}
+	}
+
+	if ( op->o_abandon ) {
+		rc = SLAPD_ABANDON;
+	} else if ( (cancel = op->o_cancel) == SLAP_CANCEL_NONE ) {
+		op->o_cancel = SLAP_CANCEL_INVALID;	/* reject cancel/abandon */
+	} else {
+		assert( cancel == SLAP_CANCEL_INVALID );
+	}
+
+	if ( need_lock ) {
+		ldap_pvt_thread_mutex_unlock( &op->o_conn->c_mutex );
+	}
+
+	return rc;
+}
Index: connection.c
===================================================================
RCS file: /repo/OpenLDAP/pkg/ldap/servers/slapd/connection.c,v
retrieving revision 1.438
diff -u -2 -r1.438 connection.c
--- connection.c	28 May 2009 15:46:25 -0000	1.438
+++ connection.c	3 Jun 2009 12:37:06 -0000
@@ -1022,5 +1022,5 @@
 connection_operation( void *ctx, void *arg_v )
 {
-	int rc = LDAP_OTHER;
+	int rc = LDAP_OTHER, cancel;
 	Operation *op = arg_v;
 	SlapReply rs = {REP_RESULT};
@@ -1126,16 +1126,18 @@
 	}
 
-	if ( op->o_cancel == SLAP_CANCEL_REQ ) {
-		if ( rc == SLAPD_ABANDON ) {
-			op->o_cancel = SLAP_CANCEL_ACK;
-		} else {
-			op->o_cancel = LDAP_TOO_LATE;
+	cancel = op->o_cancel;
+	switch ( cancel ) {
+	case SLAP_CANCEL_INVALID:
+		break;
+	case SLAP_CANCEL_REQ:
+		cancel = rc == SLAPD_ABANDON ? SLAP_CANCEL_ACK : LDAP_TOO_LATE;
+		op->o_cancel = cancel;
+		/* FALL THROUGH */
+	default:
+		while ( cancel != SLAP_CANCEL_NONE && cancel != SLAP_CANCEL_DONE ) {
+			ldap_pvt_thread_yield();
+			cancel = op->o_cancel;
 		}
-	}
-
-	while ( op->o_cancel != SLAP_CANCEL_NONE &&
-		op->o_cancel != SLAP_CANCEL_DONE )
-	{
-		ldap_pvt_thread_yield();
+		break;
 	}
 
Index: result.c
===================================================================
RCS file: /repo/OpenLDAP/pkg/ldap/servers/slapd/result.c,v
retrieving revision 1.332
diff -u -2 -r1.332 result.c
--- result.c	14 May 2009 02:03:23 -0000	1.332
+++ result.c	3 Jun 2009 12:37:06 -0000
@@ -140,5 +140,5 @@
 	ber_len_t bytes;
 	long ret = 0;
-	int closing = 0;
+	int closing = 0, cancel;
 
 	ber_get_option( ber, LBER_OPT_BER_BYTES_TO_WRITE, &bytes );
@@ -146,5 +146,8 @@
 	/* write only one pdu at a time - wait til it's our turn */
 	ldap_pvt_thread_mutex_lock( &conn->c_write1_mutex );
-	if (( op->o_abandon && !op->o_cancel ) || !connection_valid( conn )) {
+	if (( op->o_abandon
+		  && !( (cancel = op->o_cancel) && cancel != SLAP_CANCEL_INVALID ))
+		|| !connection_valid( conn ))
+	{
 		ldap_pvt_thread_mutex_unlock( &conn->c_write1_mutex );
 		return 0;
@@ -413,8 +416,10 @@
 	BerElementBuffer berbuf;
 	BerElement	*ber = (BerElement *) &berbuf;
-	int		rc = LDAP_SUCCESS;
+	int		rc = LDAP_SUCCESS, cancel;
 	long	bytes;
 
-	if (( rs->sr_err == SLAPD_ABANDON || op->o_abandon ) && !op->o_cancel ) {
+	if ( (rs->sr_err == SLAPD_ABANDON || op->o_abandon)
+		&& !( (cancel = op->o_cancel) && cancel != SLAP_CANCEL_INVALID ))
+	{
 		rc = SLAPD_ABANDON;
 		goto clean2;
@@ -439,6 +444,9 @@
 
 	rc = rs->sr_err;
-	if ( rc == SLAPD_ABANDON && op->o_cancel )
-		rc = LDAP_CANCELLED;
+	if ( rc == SLAPD_ABANDON ) {
+		rc = LDAP_OTHER;
+		if ( (cancel = op->o_cancel) && cancel != SLAP_CANCEL_INVALID )
+			rc = LDAP_CANCELLED;
+	}
 
 	Debug( LDAP_DEBUG_TRACE,
@@ -457,5 +465,5 @@
 	{
 		rc = ber_printf( ber, "t{ess" /*"}"*/,
-			rs->sr_tag, rs->sr_err,
+			rs->sr_tag, rc,
 		rs->sr_matched == NULL ? "" : rs->sr_matched,
 		rs->sr_text == NULL ? "" : rs->sr_text );
Index: abandon.c
===================================================================
RCS file: /repo/OpenLDAP/pkg/ldap/servers/slapd/abandon.c,v
retrieving revision 1.59
diff -u -2 -r1.59 abandon.c
--- abandon.c	21 May 2009 21:22:46 -0000	1.59
+++ abandon.c	3 Jun 2009 12:37:05 -0000
@@ -38,4 +38,5 @@
 	Operation	*o;
 	const char	*msg;
+	int cancel;
 
 	Debug( LDAP_DEBUG_TRACE, "%s do_abandon\n",
@@ -97,4 +98,18 @@
 			}
 		}
+
+	} else if ( o->o_tag == LDAP_REQ_BIND
+			|| o->o_tag == LDAP_REQ_UNBIND
+			|| o->o_tag == LDAP_REQ_ABANDON ) {
+		msg = "cannot be abandoned";
+
+	} else if ( (cancel = o->o_cancel) && cancel != SLAP_CANCEL_INVALID ) {
+		msg = "already being cancelled";
+
+#if 0 /* Would break o_abandon used as "suppress response" flag, ITS#6138 */
+	} else if ( o->o_abandon ) {
+		msg = "already being abandoned";
+#endif
+
 	} else {
 		msg = "found";
Index: bconfig.c
===================================================================
RCS file: /repo/OpenLDAP/pkg/ldap/servers/slapd/bconfig.c,v
retrieving revision 1.375
diff -u -2 -r1.375 bconfig.c
--- bconfig.c	2 Jun 2009 21:17:58 -0000	1.375
+++ bconfig.c	3 Jun 2009 12:37:06 -0000
@@ -4784,4 +4784,8 @@
 	}
 
+	rs->sr_err = prevent_cancel( op, 1 );
+	if ( rs->sr_err != LDAP_SUCCESS )
+		goto out;
+
 	ldap_pvt_thread_pool_pause( &connection_pool );
 
@@ -5224,6 +5228,11 @@
 	slap_mods_opattrs( op, &op->orm_modlist, 1 );
 
-	if ( do_pause )
+	if ( do_pause ) {
+		rs->sr_err = prevent_cancel( op, 1 );
+		if ( rs->sr_err != LDAP_SUCCESS )
+			goto out;
+
 		ldap_pvt_thread_pool_pause( &connection_pool );
+	}
 
 	/* Strategy:
@@ -5389,4 +5398,8 @@
 	}
 
+	rs->sr_err = prevent_cancel( op, 1 );
+	if ( rs->sr_err != LDAP_SUCCESS )
+		goto out;
+
 	ldap_pvt_thread_pool_pause( &connection_pool );
 
@@ -5482,4 +5495,8 @@
 		int count, ixold;
 
+		rs->sr_err = prevent_cancel( op, 1 );
+		if ( rs->sr_err != LDAP_SUCCESS )
+			goto out;
+
 		ldap_pvt_thread_pool_pause( &connection_pool );