22 	struct iscsi_datain_req *dr;  in iscsit_allocate_datain_req()  local
24 	dr = kmem_cache_zalloc(lio_dr_cache, GFP_ATOMIC);  in iscsit_allocate_datain_req()
25 	if (!dr) {  in iscsit_allocate_datain_req()
30 	INIT_LIST_HEAD(&dr->cmd_datain_node);  in iscsit_allocate_datain_req()
32 	return dr;  in iscsit_allocate_datain_req()
35 void iscsit_attach_datain_req(struct iscsit_cmd *cmd, struct iscsi_datain_req *dr)  in iscsit_attach_datain_req()  argument
38 	list_add_tail(&dr->cmd_datain_node, &cmd->datain_list);  in iscsit_attach_datain_req()
42 void iscsit_free_datain_req(struct iscsit_cmd *cmd, struct iscsi_datain_req *dr)  in iscsit_free_datain_req()  argument
45 	list_del(&dr->cmd_datain_node);  in iscsit_free_datain_req()
48 	kmem_cache_free(lio_dr_cache, dr);  in iscsit_free_datain_req()
53 	struct iscsi_datain_req *dr, *dr_tmp;  in iscsit_free_all_datain_reqs()  local
56 	list_for_each_entry_safe(dr, dr_tmp, &cmd->datain_list, cmd_datain_node) {  in iscsit_free_all_datain_reqs()
57 		list_del(&dr->cmd_datain_node);  in iscsit_free_all_datain_reqs()
58 		kmem_cache_free(lio_dr_cache, dr);  in iscsit_free_all_datain_reqs()
84 	struct iscsi_datain_req *dr;  in iscsit_set_datain_values_yes_and_yes()  local
86 	dr = iscsit_get_datain_req(cmd);  in iscsit_set_datain_values_yes_and_yes()
87 	if (!dr)  in iscsit_set_datain_values_yes_and_yes()
90 	if (dr->recovery && dr->generate_recovery_values) {  in iscsit_set_datain_values_yes_and_yes()
92 					cmd, dr) < 0)  in iscsit_set_datain_values_yes_and_yes()
95 		dr->generate_recovery_values = 0;  in iscsit_set_datain_values_yes_and_yes()
98 	next_burst_len = (!dr->recovery) ?  in iscsit_set_datain_values_yes_and_yes()
99 			cmd->next_burst_len : dr->next_burst_len;  in iscsit_set_datain_values_yes_and_yes()
100 	read_data_done = (!dr->recovery) ?  in iscsit_set_datain_values_yes_and_yes()
101 			cmd->read_data_done : dr->read_data_done;  in iscsit_set_datain_values_yes_and_yes()
136 	datain->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++;  in iscsit_set_datain_values_yes_and_yes()
139 	if (!dr->recovery) {  in iscsit_set_datain_values_yes_and_yes()
143 		dr->next_burst_len = next_burst_len;  in iscsit_set_datain_values_yes_and_yes()
144 		dr->read_data_done += datain->length;  in iscsit_set_datain_values_yes_and_yes()
147 	if (!dr->recovery) {  in iscsit_set_datain_values_yes_and_yes()
149 			dr->dr_complete = DATAIN_COMPLETE_NORMAL;  in iscsit_set_datain_values_yes_and_yes()
151 		return dr;  in iscsit_set_datain_values_yes_and_yes()
154 	if (!dr->runlength) {  in iscsit_set_datain_values_yes_and_yes()
156 			dr->dr_complete =  in iscsit_set_datain_values_yes_and_yes()
157 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?  in iscsit_set_datain_values_yes_and_yes()
162 		if ((dr->begrun + dr->runlength) == dr->data_sn) {  in iscsit_set_datain_values_yes_and_yes()
163 			dr->dr_complete =  in iscsit_set_datain_values_yes_and_yes()
164 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?  in iscsit_set_datain_values_yes_and_yes()
170 	return dr;  in iscsit_set_datain_values_yes_and_yes()
182 	struct iscsi_datain_req *dr;  in iscsit_set_datain_values_no_and_yes()  local
185 	dr = iscsit_get_datain_req(cmd);  in iscsit_set_datain_values_no_and_yes()
186 	if (!dr)  in iscsit_set_datain_values_no_and_yes()
189 	if (dr->recovery && dr->generate_recovery_values) {  in iscsit_set_datain_values_no_and_yes()
191 					cmd, dr) < 0)  in iscsit_set_datain_values_no_and_yes()
194 		dr->generate_recovery_values = 0;  in iscsit_set_datain_values_no_and_yes()
197 	read_data_done = (!dr->recovery) ?  in iscsit_set_datain_values_no_and_yes()
198 			cmd->read_data_done : dr->read_data_done;  in iscsit_set_datain_values_no_and_yes()
199 	seq_send_order = (!dr->recovery) ?  in iscsit_set_datain_values_no_and_yes()
200 			cmd->seq_send_order : dr->seq_send_order;  in iscsit_set_datain_values_no_and_yes()
215 	if (!dr->recovery && !seq->next_burst_len)  in iscsit_set_datain_values_no_and_yes()
257 	datain->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++;  in iscsit_set_datain_values_no_and_yes()
258 	if (!dr->recovery) {  in iscsit_set_datain_values_no_and_yes()
262 		dr->seq_send_order = seq_send_order;  in iscsit_set_datain_values_no_and_yes()
263 		dr->read_data_done += datain->length;  in iscsit_set_datain_values_no_and_yes()
266 	if (!dr->recovery) {  in iscsit_set_datain_values_no_and_yes()
270 			dr->dr_complete = DATAIN_COMPLETE_NORMAL;  in iscsit_set_datain_values_no_and_yes()
272 		return dr;  in iscsit_set_datain_values_no_and_yes()
275 	if (!dr->runlength) {  in iscsit_set_datain_values_no_and_yes()
277 			dr->dr_complete =  in iscsit_set_datain_values_no_and_yes()
278 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?  in iscsit_set_datain_values_no_and_yes()
283 		if ((dr->begrun + dr->runlength) == dr->data_sn) {  in iscsit_set_datain_values_no_and_yes()
284 			dr->dr_complete =  in iscsit_set_datain_values_no_and_yes()
285 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?  in iscsit_set_datain_values_no_and_yes()
291 	return dr;  in iscsit_set_datain_values_no_and_yes()
303 	struct iscsi_datain_req *dr;  in iscsit_set_datain_values_yes_and_no()  local
306 	dr = iscsit_get_datain_req(cmd);  in iscsit_set_datain_values_yes_and_no()
307 	if (!dr)  in iscsit_set_datain_values_yes_and_no()
310 	if (dr->recovery && dr->generate_recovery_values) {  in iscsit_set_datain_values_yes_and_no()
312 					cmd, dr) < 0)  in iscsit_set_datain_values_yes_and_no()
315 		dr->generate_recovery_values = 0;  in iscsit_set_datain_values_yes_and_no()
318 	next_burst_len = (!dr->recovery) ?  in iscsit_set_datain_values_yes_and_no()
319 			cmd->next_burst_len : dr->next_burst_len;  in iscsit_set_datain_values_yes_and_no()
320 	read_data_done = (!dr->recovery) ?  in iscsit_set_datain_values_yes_and_no()
321 			cmd->read_data_done : dr->read_data_done;  in iscsit_set_datain_values_yes_and_no()
327 		return dr;  in iscsit_set_datain_values_yes_and_no()
332 		return dr;  in iscsit_set_datain_values_yes_and_no()
353 	pdu->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++;  in iscsit_set_datain_values_yes_and_no()
354 	if (!dr->recovery) {  in iscsit_set_datain_values_yes_and_no()
358 		dr->next_burst_len = next_burst_len;  in iscsit_set_datain_values_yes_and_no()
359 		dr->read_data_done += pdu->length;  in iscsit_set_datain_values_yes_and_no()
367 	if (!dr->recovery) {  in iscsit_set_datain_values_yes_and_no()
369 			dr->dr_complete = DATAIN_COMPLETE_NORMAL;  in iscsit_set_datain_values_yes_and_no()
371 		return dr;  in iscsit_set_datain_values_yes_and_no()
374 	if (!dr->runlength) {  in iscsit_set_datain_values_yes_and_no()
376 			dr->dr_complete =  in iscsit_set_datain_values_yes_and_no()
377 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?  in iscsit_set_datain_values_yes_and_no()
382 		if ((dr->begrun + dr->runlength) == dr->data_sn) {  in iscsit_set_datain_values_yes_and_no()
383 			dr->dr_complete =  in iscsit_set_datain_values_yes_and_no()
384 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?  in iscsit_set_datain_values_yes_and_no()
390 	return dr;  in iscsit_set_datain_values_yes_and_no()
402 	struct iscsi_datain_req *dr;  in iscsit_set_datain_values_no_and_no()  local
406 	dr = iscsit_get_datain_req(cmd);  in iscsit_set_datain_values_no_and_no()
407 	if (!dr)  in iscsit_set_datain_values_no_and_no()
410 	if (dr->recovery && dr->generate_recovery_values) {  in iscsit_set_datain_values_no_and_no()
412 					cmd, dr) < 0)  in iscsit_set_datain_values_no_and_no()
415 		dr->generate_recovery_values = 0;  in iscsit_set_datain_values_no_and_no()
418 	read_data_done = (!dr->recovery) ?  in iscsit_set_datain_values_no_and_no()
419 			cmd->read_data_done : dr->read_data_done;  in iscsit_set_datain_values_no_and_no()
420 	seq_send_order = (!dr->recovery) ?  in iscsit_set_datain_values_no_and_no()
421 			cmd->seq_send_order : dr->seq_send_order;  in iscsit_set_datain_values_no_and_no()
436 	if (!dr->recovery && !seq->next_burst_len)  in iscsit_set_datain_values_no_and_no()
456 	pdu->data_sn = (!dr->recovery) ? cmd->data_sn++ : dr->data_sn++;  in iscsit_set_datain_values_no_and_no()
457 	if (!dr->recovery) {  in iscsit_set_datain_values_no_and_no()
461 		dr->seq_send_order = seq_send_order;  in iscsit_set_datain_values_no_and_no()
462 		dr->read_data_done += pdu->length;  in iscsit_set_datain_values_no_and_no()
470 	if (!dr->recovery) {  in iscsit_set_datain_values_no_and_no()
474 			dr->dr_complete = DATAIN_COMPLETE_NORMAL;  in iscsit_set_datain_values_no_and_no()
476 		return dr;  in iscsit_set_datain_values_no_and_no()
479 	if (!dr->runlength) {  in iscsit_set_datain_values_no_and_no()
481 			dr->dr_complete =  in iscsit_set_datain_values_no_and_no()
482 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?  in iscsit_set_datain_values_no_and_no()
487 		if ((dr->begrun + dr->runlength) == dr->data_sn) {  in iscsit_set_datain_values_no_and_no()
488 			dr->dr_complete =  in iscsit_set_datain_values_no_and_no()
489 			    (dr->recovery == DATAIN_WITHIN_COMMAND_RECOVERY) ?  in iscsit_set_datain_values_no_and_no()
495 	return dr;  in iscsit_set_datain_values_no_and_no()