drivers/crypto/msm/qce50.c

/* Qualcomm Crypto Engine driver.
 *
 * Copyright (c) 2012, Code Aurora Forum. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 and
 * only version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */
#define pr_fmt(fmt) "QCE50: %s: " fmt, __func__

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mod_devicetable.h>
#include <linux/device.h>
#include <linux/clk.h>
#include <linux/err.h>
#include <linux/dma-mapping.h>
#include <linux/io.h>
#include <linux/platform_device.h>
#include <linux/spinlock.h>
#include <linux/delay.h>
#include <linux/crypto.h>
#include <linux/qcedev.h>
#include <linux/bitops.h>
#include <crypto/hash.h>
#include <crypto/sha.h>
#include <mach/dma.h>
#include <mach/clk.h>
#include <mach/socinfo.h>

#include "qce.h"
#include "qce50.h"
#include "qcryptohw_50.h"

#define CRYPTO_CONFIG_RESET 0xE001F

static DEFINE_MUTEX(bam_register_cnt);
struct bam_registration_info {
	uint32_t handle;
	uint32_t cnt;
};
static struct bam_registration_info bam_registry;

/*
 * CE HW device structure.
 * Each engine has an instance of the structure.
 * Each engine can only handle one crypto operation at one time. It is up to
 * the sw above to ensure single threading of operation on an engine.
 */
struct qce_device {
	struct device *pdev;        /* Handle to platform_device structure */

	unsigned char *coh_vmem;    /* Allocated coherent virtual memory */
	dma_addr_t coh_pmem;	    /* Allocated coherent physical memory */
	int memsize;				/* Memory allocated */

	void __iomem *iobase;	    /* Virtual io base of CE HW  */
	unsigned int phy_iobase;    /* Physical io base of CE HW    */

	struct clk *ce_core_src_clk;	/* Handle to CE src clk*/
	struct clk *ce_core_clk;	/* Handle to CE clk */
	struct clk *ce_clk;		/* Handle to CE clk */

	qce_comp_func_ptr_t qce_cb;	/* qce callback function pointer */

	int assoc_nents;
	int ivsize;
	int authsize;
	int src_nents;
	int dst_nents;

	dma_addr_t phy_iv_in;

	void *areq;
	enum qce_cipher_mode_enum mode;
	struct ce_sps_data ce_sps;
};

/* Standard initialization vector for SHA-1, source: FIPS 180-2 */
static uint32_t  _std_init_vector_sha1[] =   {
	0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0
};

/* Standard initialization vector for SHA-256, source: FIPS 180-2 */
static uint32_t _std_init_vector_sha256[] = {
	0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
	0x510E527F, 0x9B05688C,	0x1F83D9AB, 0x5BE0CD19
};

static void _byte_stream_to_net_words(uint32_t *iv, unsigned char *b,
		unsigned int len)
{
	unsigned n;

	n = len  / sizeof(uint32_t) ;
	for (; n > 0; n--) {
		*iv =  ((*b << 24)      & 0xff000000) |
				(((*(b+1)) << 16) & 0xff0000)   |
				(((*(b+2)) << 8) & 0xff00)     |
				(*(b+3)          & 0xff);
		b += sizeof(uint32_t);
		iv++;
	}

	n = len %  sizeof(uint32_t);
	if (n == 3) {
		*iv = ((*b << 24) & 0xff000000) |
				(((*(b+1)) << 16) & 0xff0000)   |
				(((*(b+2)) << 8) & 0xff00)     ;
	} else if (n == 2) {
		*iv = ((*b << 24) & 0xff000000) |
				(((*(b+1)) << 16) & 0xff0000)   ;
	} else if (n == 1) {
		*iv = ((*b << 24) & 0xff000000) ;
	}
}

static void _byte_stream_swap_to_net_words(uint32_t *iv, unsigned char *b,
		unsigned int len)
{
	unsigned i, j;
	unsigned char swap_iv[AES_IV_LENGTH];

	memset(swap_iv, 0, AES_IV_LENGTH);
	for (i = (AES_IV_LENGTH-len), j = len-1;  i < AES_IV_LENGTH; i++, j--)
		swap_iv[i] = b[j];
	_byte_stream_to_net_words(iv, swap_iv, AES_IV_LENGTH);
}

static int count_sg(struct scatterlist *sg, int nbytes)
{
	int i;

	for (i = 0; nbytes > 0; i++, sg = sg_next(sg))
		nbytes -= sg->length;
	return i;
}

static int _probe_ce_engine(struct qce_device *pce_dev)
{
	unsigned int rev;
	unsigned int maj_rev, min_rev, step_rev;

	rev = readl_relaxed(pce_dev->iobase + CRYPTO_VERSION_REG);
	mb();
	maj_rev = (rev & CRYPTO_CORE_MAJOR_REV_MASK) >> CRYPTO_CORE_MAJOR_REV;
	min_rev = (rev & CRYPTO_CORE_MINOR_REV_MASK) >> CRYPTO_CORE_MINOR_REV;
	step_rev = (rev & CRYPTO_CORE_STEP_REV_MASK) >> CRYPTO_CORE_STEP_REV;

	if ((maj_rev != 0x05) || (min_rev > 0x02) || (step_rev > 0x02)) {
		pr_err("Unknown Qualcomm crypto device at 0x%x, rev %d.%d.%d\n",
			pce_dev->phy_iobase, maj_rev, min_rev, step_rev);
		return -EIO;
	};
	if ((min_rev > 0)  && (step_rev != 0)) {
		pr_err("Unknown Qualcomm crypto device at 0x%x, rev %d.%d.%d\n",
			pce_dev->phy_iobase, maj_rev, min_rev, step_rev);
		return -EIO;
	};
	pce_dev->ce_sps.minor_version = min_rev;

	dev_info(pce_dev->pdev, "Qualcomm Crypto %d.%d.%d device found @0x%x\n",
			maj_rev, min_rev, step_rev, pce_dev->phy_iobase);

	pce_dev->ce_sps.ce_burst_size = MAX_CE_BAM_BURST_SIZE;

	dev_info(pce_dev->pdev,
			"IO base, CE = 0x%x\n, "
			"Consumer (IN) PIPE %d,    "
			"Producer (OUT) PIPE %d\n"
			"IO base BAM = 0x%x\n"
			"BAM IRQ %d\n",
			(uint32_t) pce_dev->iobase,
			pce_dev->ce_sps.dest_pipe_index,
			pce_dev->ce_sps.src_pipe_index,
			(uint32_t)pce_dev->ce_sps.bam_iobase,
			pce_dev->ce_sps.bam_irq);
	return 0;
};

static int _ce_get_hash_cmdlistinfo(struct qce_device *pce_dev,
				struct qce_sha_req *sreq,
				struct qce_cmdlist_info **cmdplistinfo)
{
	struct qce_cmdlistptr_ops *cmdlistptr = &pce_dev->ce_sps.cmdlistptr;

	switch (sreq->alg) {
	case QCE_HASH_SHA1:
		*cmdplistinfo = &cmdlistptr->auth_sha1;
		break;

	case QCE_HASH_SHA256:
		*cmdplistinfo = &cmdlistptr->auth_sha256;
		break;

	case QCE_HASH_SHA1_HMAC:
		*cmdplistinfo = &cmdlistptr->auth_sha1_hmac;
			break;

	case QCE_HASH_SHA256_HMAC:
		*cmdplistinfo = &cmdlistptr->auth_sha256_hmac;
		break;

	case QCE_HASH_AES_CMAC:
		if (sreq->authklen == AES128_KEY_SIZE)
			*cmdplistinfo = &cmdlistptr->auth_aes_128_cmac;
		else
			*cmdplistinfo = &cmdlistptr->auth_aes_256_cmac;
		break;

	default:
		break;
	}
	return 0;
}

static int _ce_setup_hash(struct qce_device *pce_dev,
				struct qce_sha_req *sreq,
				struct qce_cmdlist_info *cmdlistinfo)
{
	uint32_t auth32[SHA256_DIGEST_SIZE / sizeof(uint32_t)];
	uint32_t diglen;
	int i;
	uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = {
			0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
	bool sha1 = false;
	struct sps_command_element *pce = NULL;

	if ((sreq->alg == QCE_HASH_SHA1_HMAC) ||
			(sreq->alg == QCE_HASH_SHA256_HMAC) ||
			(sreq->alg ==  QCE_HASH_AES_CMAC)) {
		uint32_t authk_size_in_word = sreq->authklen/sizeof(uint32_t);

		_byte_stream_to_net_words(mackey32, sreq->authkey,
						sreq->authklen);

		/* check for null key. If null, use hw key*/
		for (i = 0; i < authk_size_in_word; i++) {
			if (mackey32[i] != 0)
				break;
		}

		pce = cmdlistinfo->go_proc;
		if (i == authk_size_in_word) {
			pce->addr = (uint32_t)(CRYPTO_GOPROC_OEM_KEY_REG +
							pce_dev->phy_iobase);
		} else {
			pce->addr = (uint32_t)(CRYPTO_GOPROC_REG +
							pce_dev->phy_iobase);
			pce = cmdlistinfo->auth_key;
			for (i = 0; i < authk_size_in_word; i++, pce++)
				pce->data = mackey32[i];
		}
	}

	if (sreq->alg ==  QCE_HASH_AES_CMAC)
		goto go_proc;

	/* if not the last, the size has to be on the block boundary */
	if (sreq->last_blk == 0 && (sreq->size % SHA256_BLOCK_SIZE))
		return -EIO;

	switch (sreq->alg) {
	case QCE_HASH_SHA1:
	case QCE_HASH_SHA1_HMAC:
		diglen = SHA1_DIGEST_SIZE;
		sha1 = true;
		break;
	case QCE_HASH_SHA256:
	case QCE_HASH_SHA256_HMAC:
		diglen = SHA256_DIGEST_SIZE;
		break;
	default:
		return -EINVAL;
	}

	/* write 20/32 bytes, 5/8 words into auth_iv for SHA1/SHA256 */
	if (sreq->first_blk) {
		if (sha1) {
			for (i = 0; i < 5; i++)
				auth32[i] = _std_init_vector_sha1[i];
		} else {
			for (i = 0; i < 8; i++)
				auth32[i] = _std_init_vector_sha256[i];
		}
	} else {
		_byte_stream_to_net_words(auth32, sreq->digest, diglen);
	}

	pce = cmdlistinfo->auth_iv;
	for (i = 0; i < 5; i++, pce++)
		pce->data = auth32[i];

	if ((sreq->alg == QCE_HASH_SHA256) ||
			(sreq->alg == QCE_HASH_SHA256_HMAC)) {
		for (i = 5; i < 8; i++, pce++)
			pce->data = auth32[i];
	}

	/* write auth_bytecnt 0/1, start with 0 */
	pce = cmdlistinfo->auth_bytecount;
	for (i = 0; i < 2; i++, pce++)
		pce->data = sreq->auth_data[i];

	/* Set/reset  last bit in CFG register  */
	pce = cmdlistinfo->auth_seg_cfg;
	if (sreq->last_blk)
		pce->data |= 1 << CRYPTO_LAST;
	else
		pce->data &= ~(1 << CRYPTO_LAST);
	if (sreq->first_blk)
		pce->data |= 1 << CRYPTO_FIRST;
	else
		pce->data &= ~(1 << CRYPTO_FIRST);
go_proc:
	/* write auth seg size */
	pce = cmdlistinfo->auth_seg_size;
	pce->data = sreq->size;

	/* write auth seg size start*/
	pce = cmdlistinfo->auth_seg_start;
	pce->data = 0;

	/* write seg size */
	pce = cmdlistinfo->seg_size;
	pce->data = sreq->size;

	return 0;
}

static int _ce_get_cipher_cmdlistinfo(struct qce_device *pce_dev,
				struct qce_req *creq,
				struct qce_cmdlist_info **cmdlistinfo)
{
	struct qce_cmdlistptr_ops *cmdlistptr = &pce_dev->ce_sps.cmdlistptr;

	if (creq->alg != CIPHER_ALG_AES) {
		switch (creq->alg) {
		case CIPHER_ALG_DES:
			if (creq->mode ==  QCE_MODE_ECB)
				*cmdlistinfo = &cmdlistptr->cipher_des_ecb;
			else
				*cmdlistinfo = &cmdlistptr->cipher_des_cbc;
			break;

		case CIPHER_ALG_3DES:
			if (creq->mode ==  QCE_MODE_ECB)
				*cmdlistinfo =
					&cmdlistptr->cipher_3des_ecb;
			else
				*cmdlistinfo =
					&cmdlistptr->cipher_3des_cbc;
			break;
		default:
			break;
		}
	} else {
		switch (creq->mode) {
		case QCE_MODE_ECB:
			if (creq->encklen ==  AES128_KEY_SIZE)
				*cmdlistinfo = &cmdlistptr->cipher_aes_128_ecb;
			else
				*cmdlistinfo = &cmdlistptr->cipher_aes_256_ecb;
			break;

		case QCE_MODE_CBC:
		case QCE_MODE_CTR:
			if (creq->encklen ==  AES128_KEY_SIZE)
				*cmdlistinfo =
					&cmdlistptr->cipher_aes_128_cbc_ctr;
			else
				*cmdlistinfo =
					&cmdlistptr->cipher_aes_256_cbc_ctr;
			break;

		case QCE_MODE_XTS:
			if (creq->encklen ==  AES128_KEY_SIZE)
				*cmdlistinfo = &cmdlistptr->cipher_aes_128_xts;
			else
				*cmdlistinfo = &cmdlistptr->cipher_aes_256_xts;
			break;

		case QCE_MODE_CCM:
			if (creq->encklen ==  AES128_KEY_SIZE)
				*cmdlistinfo = &cmdlistptr->aead_aes_128_ccm;
			else
				*cmdlistinfo = &cmdlistptr->aead_aes_256_ccm;
			break;

		default:
			break;
		}
	}
	return 0;
}

static int _ce_setup_cipher(struct qce_device *pce_dev, struct qce_req *creq,
		uint32_t totallen_in, uint32_t coffset,
		struct qce_cmdlist_info *cmdlistinfo)
{
	uint32_t enckey32[(MAX_CIPHER_KEY_SIZE * 2)/sizeof(uint32_t)] = {
			0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
	uint32_t enciv32[MAX_IV_LENGTH / sizeof(uint32_t)] = {
			0, 0, 0, 0};
	uint32_t enck_size_in_word = 0;
	uint32_t key_size;
	bool use_hw_key = false;
	uint32_t encr_cfg = 0;
	uint32_t ivsize = creq->ivsize;
	int i;
	struct sps_command_element *pce = NULL;

	if (creq->mode == QCE_MODE_XTS)
		key_size = creq->encklen/2;
	else
		key_size = creq->encklen;

	_byte_stream_to_net_words(enckey32, creq->enckey, key_size);

	/* check for null key. If null, use hw key*/
	enck_size_in_word = key_size/sizeof(uint32_t);
	for (i = 0; i < enck_size_in_word; i++) {
		if (enckey32[i] != 0)
			break;
	}
	pce = cmdlistinfo->go_proc;
	if (i == enck_size_in_word) {
		use_hw_key = true;
		pce->addr = (uint32_t)(CRYPTO_GOPROC_OEM_KEY_REG +
						pce_dev->phy_iobase);
	} else {
		pce->addr = (uint32_t)(CRYPTO_GOPROC_REG +
						pce_dev->phy_iobase);
	}

	if ((creq->op == QCE_REQ_AEAD) && (creq->mode == QCE_MODE_CCM)) {
		uint32_t authklen32 = creq->encklen/sizeof(uint32_t);
		uint32_t noncelen32 = MAX_NONCE/sizeof(uint32_t);
		uint32_t nonce32[MAX_NONCE/sizeof(uint32_t)] = {0, 0, 0, 0};
		uint32_t auth_cfg = 0;

		/* write nonce */
		_byte_stream_to_net_words(nonce32, creq->nonce, MAX_NONCE);
		pce = cmdlistinfo->auth_nonce_info;
		for (i = 0; i < noncelen32; i++, pce++)
			pce->data = nonce32[i];

		/* TBD  NEW FEATURE partial AES CCM  pkt support set last bit */
		auth_cfg |= ((1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST));
		if (creq->dir == QCE_ENCRYPT)
			auth_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
		else
			auth_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS);
		auth_cfg |= ((creq->authsize - 1) << CRYPTO_AUTH_SIZE);
		auth_cfg |= (CRYPTO_AUTH_MODE_CCM << CRYPTO_AUTH_MODE);
		if (creq->authklen ==  AES128_KEY_SIZE)
			auth_cfg |= (CRYPTO_AUTH_KEY_SZ_AES128 <<
						CRYPTO_AUTH_KEY_SIZE);
		else {
			if (creq->authklen ==  AES256_KEY_SIZE)
				auth_cfg |= (CRYPTO_AUTH_KEY_SZ_AES256 <<
							CRYPTO_AUTH_KEY_SIZE);
		}
		auth_cfg |= (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG);
		auth_cfg |= ((MAX_NONCE/sizeof(uint32_t)) <<
						CRYPTO_AUTH_NONCE_NUM_WORDS);

		if (use_hw_key == true)	{
			auth_cfg |= (1 << CRYPTO_USE_HW_KEY_AUTH);
		} else {
			auth_cfg &= ~(1 << CRYPTO_USE_HW_KEY_AUTH);
			/* write auth key */
			pce = cmdlistinfo->auth_key;
			for (i = 0; i < authklen32; i++, pce++)
				pce->data = enckey32[i];
		}

		pce = cmdlistinfo->auth_seg_cfg;
		pce->data = auth_cfg;

		pce = cmdlistinfo->auth_seg_size;
		pce->data = totallen_in;
		pce = cmdlistinfo->auth_seg_start;
		pce->data = 0;
	}

	switch (creq->mode) {
	case QCE_MODE_ECB:
		encr_cfg |= (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE);
		break;
	case QCE_MODE_CBC:
		encr_cfg |= (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE);
		break;
	case QCE_MODE_XTS:
		encr_cfg |= (CRYPTO_ENCR_MODE_XTS << CRYPTO_ENCR_MODE);
		break;
	case QCE_MODE_CCM:
		encr_cfg |= (CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE);
		break;
	case QCE_MODE_CTR:
	default:
		encr_cfg |= (CRYPTO_ENCR_MODE_CTR << CRYPTO_ENCR_MODE);
		break;
	}
	pce_dev->mode = creq->mode;

	switch (creq->alg) {
	case CIPHER_ALG_DES:
		if (creq->mode !=  QCE_MODE_ECB) {
			_byte_stream_to_net_words(enciv32, creq->iv, ivsize);
			pce = cmdlistinfo->encr_cntr_iv;
			pce->data = enciv32[0];
			pce++;
			pce->data = enciv32[1];
		}
		if (use_hw_key == false) {
			pce = cmdlistinfo->encr_key;
			pce->data = enckey32[0];
			pce++;
			pce->data = enckey32[1];
		}
		break;
	case CIPHER_ALG_3DES:
		if (creq->mode !=  QCE_MODE_ECB) {
			_byte_stream_to_net_words(enciv32, creq->iv, ivsize);
			pce = cmdlistinfo->encr_cntr_iv;
			pce->data = enciv32[0];
			pce++;
			pce->data = enciv32[1];
		}
		if (use_hw_key == false) {
			/* write encr key */
			pce = cmdlistinfo->encr_key;
			for (i = 0; i < 6; i++, pce++)
				pce->data = enckey32[i];
		}
		break;
	case CIPHER_ALG_AES:
	default:
		if (creq->mode ==  QCE_MODE_XTS) {
			uint32_t xtskey32[MAX_CIPHER_KEY_SIZE/sizeof(uint32_t)]
					= {0, 0, 0, 0, 0, 0, 0, 0};
			uint32_t xtsklen =
					creq->encklen/(2 * sizeof(uint32_t));

			_byte_stream_to_net_words(xtskey32, (creq->enckey +
					creq->encklen/2), creq->encklen/2);
			/* write xts encr key */
			pce = cmdlistinfo->encr_xts_key;
			for (i = 0; i < xtsklen; i++, pce++)
				pce->data = xtskey32[i];

			/* write xts du size */
			pce = cmdlistinfo->encr_xts_du_size;
			pce->data = creq->cryptlen;
		}
		if (creq->mode !=  QCE_MODE_ECB) {
			if (creq->mode ==  QCE_MODE_XTS)
				_byte_stream_swap_to_net_words(enciv32,
							creq->iv, ivsize);
			else
				_byte_stream_to_net_words(enciv32, creq->iv,
								ivsize);
			/* write encr cntr iv */
			pce = cmdlistinfo->encr_cntr_iv;
			for (i = 0; i < 4; i++, pce++)
				pce->data = enciv32[i];

			if (creq->mode ==  QCE_MODE_CCM) {
				/* write cntr iv for ccm */
				pce = cmdlistinfo->encr_ccm_cntr_iv;
				for (i = 0; i < 4; i++, pce++)
					pce->data = enciv32[i];
				/* update cntr_iv[3] by one */
				pce = cmdlistinfo->encr_cntr_iv;
				pce += 3;
				pce->data += 1;
			}
		}

		if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) {
				encr_cfg |= (CRYPTO_ENCR_KEY_SZ_AES128 <<
						CRYPTO_ENCR_KEY_SZ);
			encr_cfg |= CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG;
		} else {
			if (use_hw_key == false) {
				/* write encr key */
				pce = cmdlistinfo->encr_key;
				for (i = 0; i < enck_size_in_word; i++, pce++)
					pce->data = enckey32[i];
				switch (key_size) {
				case AES128_KEY_SIZE:
					encr_cfg |= (CRYPTO_ENCR_KEY_SZ_AES128
							 << CRYPTO_ENCR_KEY_SZ);
					break;
				case AES256_KEY_SIZE:
				default:
					encr_cfg |= (CRYPTO_ENCR_KEY_SZ_AES256
							 << CRYPTO_ENCR_KEY_SZ);
				break;
				} /* end of switch (creq->encklen) */
			}
			encr_cfg |= CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG;
		} /* else of if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) */
		break;
	} /* end of switch (creq->mode)  */

	/* write encr seg cfg */
	pce = cmdlistinfo->encr_seg_cfg;
	if ((creq->alg == CIPHER_ALG_DES) || (creq->alg == CIPHER_ALG_3DES)) {
		if (creq->dir == QCE_ENCRYPT)
			pce->data |= (1 << CRYPTO_ENCODE);
		else
			pce->data &= ~(1 << CRYPTO_ENCODE);
		encr_cfg = pce->data;
	}  else	{
		encr_cfg |=
			((creq->dir == QCE_ENCRYPT) ? 1 : 0) << CRYPTO_ENCODE;
	}
	if (use_hw_key == true)
		encr_cfg |= (CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR);
	else
		encr_cfg &= ~(CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR);
	pce->data = encr_cfg;

	/* write encr seg size */
	pce = cmdlistinfo->encr_seg_size;
	if ((creq->mode == QCE_MODE_CCM) && (creq->dir == QCE_DECRYPT))
		pce->data = (creq->cryptlen + creq->authsize);
	else
		pce->data = creq->cryptlen;

	/* write encr seg start */
	pce = cmdlistinfo->encr_seg_start;
	pce->data = (coffset & 0xffff);

	/* write seg size  */
	pce = cmdlistinfo->seg_size;
	pce->data = totallen_in;

	return 0;
};

static int _aead_complete(struct qce_device *pce_dev)
{
	struct aead_request *areq;
	unsigned char mac[SHA256_DIGEST_SIZE];

	areq = (struct aead_request *) pce_dev->areq;
	if (areq->src != areq->dst) {
		dma_unmap_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents,
					DMA_FROM_DEVICE);
	}
	dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents,
			(areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
							DMA_TO_DEVICE);
	dma_unmap_sg(pce_dev->pdev, areq->assoc, pce_dev->assoc_nents,
			DMA_TO_DEVICE);
	/* check MAC */
	memcpy(mac, (char *)(&pce_dev->ce_sps.result->auth_iv[0]),
						SHA256_DIGEST_SIZE);
	if (pce_dev->mode == QCE_MODE_CCM) {
		uint32_t result_status;
		result_status = pce_dev->ce_sps.result->status;
		result_status &= (1 << CRYPTO_MAC_FAILED);
		result_status |= (pce_dev->ce_sps.consumer_status |
					pce_dev->ce_sps.producer_status);
		pce_dev->qce_cb(areq, mac, NULL, result_status);
	} else {
		uint32_t ivsize = 0;
		struct crypto_aead *aead;
		unsigned char iv[NUM_OF_CRYPTO_CNTR_IV_REG * CRYPTO_REG_SIZE];

		aead = crypto_aead_reqtfm(areq);
		ivsize = crypto_aead_ivsize(aead);
		dma_unmap_single(pce_dev->pdev, pce_dev->phy_iv_in,
				ivsize, DMA_TO_DEVICE);
		memcpy(iv, (char *)(pce_dev->ce_sps.result->encr_cntr_iv),
			sizeof(iv));
		pce_dev->qce_cb(areq, mac, iv, pce_dev->ce_sps.consumer_status |
			pce_dev->ce_sps.producer_status);

	}
	return 0;
};

static void _sha_complete(struct qce_device *pce_dev)
{
	struct ahash_request *areq;
	unsigned char digest[SHA256_DIGEST_SIZE];

	areq = (struct ahash_request *) pce_dev->areq;
	dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents,
				DMA_TO_DEVICE);
	memcpy(digest, (char *)(&pce_dev->ce_sps.result->auth_iv[0]),
						SHA256_DIGEST_SIZE);
	pce_dev->qce_cb(areq, digest,
			(char *)pce_dev->ce_sps.result->auth_byte_count,
				pce_dev->ce_sps.consumer_status);
};

static int _ablk_cipher_complete(struct qce_device *pce_dev)
{
	struct ablkcipher_request *areq;
	unsigned char iv[NUM_OF_CRYPTO_CNTR_IV_REG * CRYPTO_REG_SIZE];

	areq = (struct ablkcipher_request *) pce_dev->areq;

	if (areq->src != areq->dst) {
		dma_unmap_sg(pce_dev->pdev, areq->dst,
			pce_dev->dst_nents, DMA_FROM_DEVICE);
	}
	dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents,
		(areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
						DMA_TO_DEVICE);

	if (pce_dev->mode == QCE_MODE_ECB) {
		pce_dev->qce_cb(areq, NULL, NULL,
					pce_dev->ce_sps.consumer_status |
					pce_dev->ce_sps.producer_status);
	} else {
		if (pce_dev->ce_sps.minor_version == 0) {
			if (pce_dev->mode == QCE_MODE_CBC)
				memcpy(iv, (char *)sg_virt(areq->src),
							sizeof(iv));

			if ((pce_dev->mode == QCE_MODE_CTR) ||
				(pce_dev->mode == QCE_MODE_XTS)) {
				uint32_t num_blk = 0;
				uint32_t cntr_iv = 0;

				memcpy(iv, areq->info, sizeof(iv));
				if (pce_dev->mode == QCE_MODE_CTR)
					num_blk = areq->nbytes/16;
				cntr_iv = (u32)(((u32)(*(iv + 14))) << 8) |
							(u32)(*(iv + 15));
				*(iv + 14) = (char)((cntr_iv + num_blk) >> 8);
				*(iv + 15) = (char)((cntr_iv + num_blk) & 0xFF);
			}
		} else {
			memcpy(iv,
				(char *)(pce_dev->ce_sps.result->encr_cntr_iv),
				sizeof(iv));
		}
		pce_dev->qce_cb(areq, NULL, iv,
			pce_dev->ce_sps.consumer_status |
			pce_dev->ce_sps.producer_status);
	}
	return 0;
};

#ifdef QCE_DEBUG
static void _qce_dump_descr_fifos(struct qce_device *pce_dev)
{
	int i, j, ents;
	struct sps_iovec *iovec = pce_dev->ce_sps.in_transfer.iovec;
	uint32_t cmd_flags = SPS_IOVEC_FLAG_CMD | SPS_IOVEC_FLAG_NWD;

	printk(KERN_INFO "==============================================\n");
	printk(KERN_INFO "CONSUMER (TX/IN/DEST) PIPE DESCRIPTOR\n");
	printk(KERN_INFO "==============================================\n");
	for (i = 0; i <  pce_dev->ce_sps.in_transfer.iovec_count; i++) {
		printk(KERN_INFO " [%d] addr=0x%x  size=0x%x  flags=0x%x\n", i,
					iovec->addr, iovec->size, iovec->flags);
		if (iovec->flags & cmd_flags) {
			struct sps_command_element *pced;

			pced = (struct sps_command_element *)
					(GET_VIRT_ADDR(iovec->addr));
			ents = iovec->size/(sizeof(struct sps_command_element));
			for (j = 0; j < ents; j++) {
				printk(KERN_INFO "      [%d] [0x%x] 0x%x\n", j,
					pced->addr, pced->data);
				pced++;
			}
		}
		iovec++;
	}

	printk(KERN_INFO "==============================================\n");
	printk(KERN_INFO "PRODUCER (RX/OUT/SRC) PIPE DESCRIPTOR\n");
	printk(KERN_INFO "==============================================\n");
	iovec = pce_dev->ce_sps.out_transfer.iovec;
	for (i = 0; i <  pce_dev->ce_sps.out_transfer.iovec_count; i++) {
		printk(KERN_INFO " [%d] addr=0x%x  size=0x%x  flags=0x%x\n", i,
				iovec->addr, iovec->size, iovec->flags);
		iovec++;
	}
}

#else
static void _qce_dump_descr_fifos(struct qce_device *pce_dev)
{
}
#endif

static void _qce_sps_iovec_count_init(struct qce_device *pce_dev)
{
	pce_dev->ce_sps.in_transfer.iovec_count = 0;
	pce_dev->ce_sps.out_transfer.iovec_count = 0;
}

static void _qce_set_eot_flag(struct sps_transfer *sps_bam_pipe)
{
	struct sps_iovec *iovec = sps_bam_pipe->iovec +
					(sps_bam_pipe->iovec_count - 1);
	iovec->flags |= SPS_IOVEC_FLAG_EOT;
}

static void _qce_sps_add_data(uint32_t addr, uint32_t len,
		struct sps_transfer *sps_bam_pipe)
{
	struct sps_iovec *iovec = sps_bam_pipe->iovec +
					sps_bam_pipe->iovec_count;
	if (len) {
		iovec->size = len;
		iovec->addr = addr;
		iovec->flags = 0;
		sps_bam_pipe->iovec_count++;
	}
}

static int _qce_sps_add_sg_data(struct qce_device *pce_dev,
		struct scatterlist *sg_src, uint32_t nbytes,
		struct sps_transfer *sps_bam_pipe)
{
	uint32_t addr, data_cnt, len;
	struct sps_iovec *iovec = sps_bam_pipe->iovec +
						sps_bam_pipe->iovec_count;

	while (nbytes > 0) {
		len = min(nbytes, sg_dma_len(sg_src));
		nbytes -= len;
		addr = sg_dma_address(sg_src);
		if (pce_dev->ce_sps.minor_version == 0)
			len = ALIGN(len, pce_dev->ce_sps.ce_burst_size);
		while (len > 0) {
			if (len > SPS_MAX_PKT_SIZE) {
				data_cnt = SPS_MAX_PKT_SIZE;
				iovec->size = data_cnt;
				iovec->addr = addr;
				iovec->flags = 0;
			} else {
				data_cnt = len;
				iovec->size = data_cnt;
				iovec->addr = addr;
				iovec->flags = 0;
			}
			iovec++;
			sps_bam_pipe->iovec_count++;
			addr += data_cnt;
			len -= data_cnt;
		}
		sg_src++;
	}
	return 0;
}

static int _qce_sps_add_cmd(struct qce_device *pce_dev, uint32_t flag,
				struct qce_cmdlist_info *cmdptr,
				struct sps_transfer *sps_bam_pipe)
{
	struct sps_iovec *iovec = sps_bam_pipe->iovec +
					sps_bam_pipe->iovec_count;
	iovec->size = cmdptr->size;
	iovec->addr = GET_PHYS_ADDR(cmdptr->cmdlist);
	iovec->flags = SPS_IOVEC_FLAG_CMD | SPS_IOVEC_FLAG_NWD | flag;
	sps_bam_pipe->iovec_count++;

	return 0;
}

static int _qce_sps_transfer(struct qce_device *pce_dev)
{
	int rc = 0;

	_qce_dump_descr_fifos(pce_dev);
	rc = sps_transfer(pce_dev->ce_sps.consumer.pipe,
					  &pce_dev->ce_sps.in_transfer);
	if (rc) {
		pr_err("sps_xfr() fail (consumer pipe=0x%x) rc = %d,",
				(u32)pce_dev->ce_sps.consumer.pipe, rc);
		return rc;
	}
	rc = sps_transfer(pce_dev->ce_sps.producer.pipe,
					  &pce_dev->ce_sps.out_transfer);
	if (rc) {
		pr_err("sps_xfr() fail (producer pipe=0x%x) rc = %d,",
				(u32)pce_dev->ce_sps.producer.pipe, rc);
		return rc;
	}
	return rc;
}

/**
 * Allocate and Connect a CE peripheral's SPS endpoint
 *
 * This function allocates endpoint context and
 * connect it with memory endpoint by calling
 * appropriate SPS driver APIs.
 *
 * Also registers a SPS callback function with
 * SPS driver
 *
 * This function should only be called once typically
 * during driver probe.
 *
 * @pce_dev - Pointer to qce_device structure
 * @ep   - Pointer to sps endpoint data structure
 * @is_produce - 1 means Producer endpoint
 *		 0 means Consumer endpoint
 *
 * @return - 0 if successful else negative value.
 *
 */
static int qce_sps_init_ep_conn(struct qce_device *pce_dev,
				struct qce_sps_ep_conn_data *ep,
				bool is_producer)
{
	int rc = 0;
	struct sps_pipe *sps_pipe_info;
	struct sps_connect *sps_connect_info = &ep->connect;
	struct sps_register_event *sps_event = &ep->event;

	/* Allocate endpoint context */
	sps_pipe_info = sps_alloc_endpoint();
	if (!sps_pipe_info) {
		pr_err("sps_alloc_endpoint() failed!!! is_producer=%d",
			   is_producer);
		rc = -ENOMEM;
		goto out;
	}
	/* Now save the sps pipe handle */
	ep->pipe = sps_pipe_info;

	/* Get default connection configuration for an endpoint */
	rc = sps_get_config(sps_pipe_info, sps_connect_info);
	if (rc) {
		pr_err("sps_get_config() fail pipe_handle=0x%x, rc = %d\n",
			(u32)sps_pipe_info, rc);
		goto get_config_err;
	}

	/* Modify the default connection configuration */
	if (is_producer) {
		/*
		* For CE producer transfer, source should be
		* CE peripheral where as destination should
		* be system memory.
		*/
		sps_connect_info->source = pce_dev->ce_sps.bam_handle;
		sps_connect_info->destination = SPS_DEV_HANDLE_MEM;
		/* Producer pipe will handle this connection */
		sps_connect_info->mode = SPS_MODE_SRC;
		sps_connect_info->options =
			SPS_O_AUTO_ENABLE | SPS_O_EOT;
	} else {
		/* For CE consumer transfer, source should be
		 * system memory where as destination should
		 * CE peripheral
		 */
		sps_connect_info->source = SPS_DEV_HANDLE_MEM;
		sps_connect_info->destination = pce_dev->ce_sps.bam_handle;
		sps_connect_info->mode = SPS_MODE_DEST;
		sps_connect_info->options =
			SPS_O_AUTO_ENABLE | SPS_O_EOT;
	}

	/* Producer pipe index */
	sps_connect_info->src_pipe_index = pce_dev->ce_sps.src_pipe_index;
	/* Consumer pipe index */
	sps_connect_info->dest_pipe_index = pce_dev->ce_sps.dest_pipe_index;
	sps_connect_info->event_thresh = 0x10;
	/*
	 * Max. no of scatter/gather buffers that can
	 * be passed by block layer = 32 (NR_SG).
	 * Each BAM descritor needs 64 bits (8 bytes).
	 * One BAM descriptor is required per buffer transfer.
	 * So we would require total 256 (32 * 8) bytes of descriptor FIFO.
	 * But due to HW limitation we need to allocate atleast one extra
	 * descriptor memory (256 bytes + 8 bytes). But in order to be
	 * in power of 2, we are allocating 512 bytes of memory.
	 */
	sps_connect_info->desc.size = 512;
	sps_connect_info->desc.base = dma_alloc_coherent(pce_dev->pdev,
					sps_connect_info->desc.size,
					&sps_connect_info->desc.phys_base,
					GFP_KERNEL);
	if (sps_connect_info->desc.base == NULL) {
		rc = -ENOMEM;
		pr_err("Can not allocate coherent memory for sps data\n");
		goto get_config_err;
	}

	memset(sps_connect_info->desc.base, 0x00, sps_connect_info->desc.size);

	/* Establish connection between peripheral and memory endpoint */
	rc = sps_connect(sps_pipe_info, sps_connect_info);
	if (rc) {
		pr_err("sps_connect() fail pipe_handle=0x%x, rc = %d\n",
			(u32)sps_pipe_info, rc);
		goto sps_connect_err;
	}

	sps_event->mode = SPS_TRIGGER_CALLBACK;
	sps_event->options = SPS_O_EOT;
	sps_event->xfer_done = NULL;
	sps_event->user = (void *)pce_dev;

	pr_debug("success, %s : pipe_handle=0x%x, desc fifo base (phy) = 0x%x\n",
		is_producer ? "PRODUCER(RX/OUT)" : "CONSUMER(TX/IN)",
		(u32)sps_pipe_info, sps_connect_info->desc.phys_base);
	goto out;

sps_connect_err:
	dma_free_coherent(pce_dev->pdev,
			sps_connect_info->desc.size,
			sps_connect_info->desc.base,
			sps_connect_info->desc.phys_base);
get_config_err:
	sps_free_endpoint(sps_pipe_info);
out:
	return rc;
}

/**
 * Disconnect and Deallocate a CE peripheral's SPS endpoint
 *
 * This function disconnect endpoint and deallocates
 * endpoint context.
 *
 * This function should only be called once typically
 * during driver remove.
 *
 * @pce_dev - Pointer to qce_device structure
 * @ep   - Pointer to sps endpoint data structure
 *
 */
static void qce_sps_exit_ep_conn(struct qce_device *pce_dev,
				struct qce_sps_ep_conn_data *ep)
{
	struct sps_pipe *sps_pipe_info = ep->pipe;
	struct sps_connect *sps_connect_info = &ep->connect;

	sps_disconnect(sps_pipe_info);
	dma_free_coherent(pce_dev->pdev,
			sps_connect_info->desc.size,
			sps_connect_info->desc.base,
			sps_connect_info->desc.phys_base);
	sps_free_endpoint(sps_pipe_info);
}
/**
 * Initialize SPS HW connected with CE core
 *
 * This function register BAM HW resources with
 * SPS driver and then initialize 2 SPS endpoints
 *
 * This function should only be called once typically
 * during driver probe.
 *
 * @pce_dev - Pointer to qce_device structure
 *
 * @return - 0 if successful else negative value.
 *
 */
static int qce_sps_init(struct qce_device *pce_dev)
{
	int rc = 0;
	struct sps_bam_props bam = {0};
	bool register_bam = false;

	bam.phys_addr = pce_dev->ce_sps.bam_mem;
	bam.virt_addr = pce_dev->ce_sps.bam_iobase;

	/*
	 * This event thresold value is only significant for BAM-to-BAM
	 * transfer. It's ignored for BAM-to-System mode transfer.
	 */
	bam.event_threshold = 0x10;	/* Pipe event threshold */
	/*
	 * This threshold controls when the BAM publish
	 * the descriptor size on the sideband interface.
	 * SPS HW will only be used when
	 * data transfer size >  64 bytes.
	 */
	bam.summing_threshold = 64;
	/* SPS driver wll handle the crypto BAM IRQ */
	bam.irq = (u32)pce_dev->ce_sps.bam_irq;
	bam.manage = SPS_BAM_MGR_LOCAL;

	pr_debug("bam physical base=0x%x\n", (u32)bam.phys_addr);
	pr_debug("bam virtual base=0x%x\n", (u32)bam.virt_addr);

	mutex_lock(&bam_register_cnt);
	if ((bam_registry.handle == 0) && (bam_registry.cnt == 0)) {
		/* Register CE Peripheral BAM device to SPS driver */
		rc = sps_register_bam_device(&bam, &bam_registry.handle);
		if (rc) {
			pr_err("sps_register_bam_device() failed! err=%d", rc);
			return -EIO;
		}
		bam_registry.cnt++;
		register_bam = true;
	} else {
		   bam_registry.cnt++;
	}
	mutex_unlock(&bam_register_cnt);
	pce_dev->ce_sps.bam_handle =  bam_registry.handle;
	pr_debug("BAM device registered. bam_handle=0x%x",
		pce_dev->ce_sps.bam_handle);

	rc = qce_sps_init_ep_conn(pce_dev, &pce_dev->ce_sps.producer, true);
	if (rc)
		goto sps_connect_producer_err;
	rc = qce_sps_init_ep_conn(pce_dev, &pce_dev->ce_sps.consumer, false);
	if (rc)
		goto sps_connect_consumer_err;

	pce_dev->ce_sps.out_transfer.user = pce_dev->ce_sps.producer.pipe;
	pce_dev->ce_sps.in_transfer.user = pce_dev->ce_sps.consumer.pipe;
	pr_info(" Qualcomm MSM CE-BAM at 0x%016llx irq %d\n",
		(unsigned long long)pce_dev->ce_sps.bam_mem,
		(unsigned int)pce_dev->ce_sps.bam_irq);
	return rc;

sps_connect_consumer_err:
	qce_sps_exit_ep_conn(pce_dev, &pce_dev->ce_sps.producer);
sps_connect_producer_err:
	if (register_bam)
		sps_deregister_bam_device(pce_dev->ce_sps.bam_handle);

	return rc;
}

/**
 * De-initialize SPS HW connected with CE core
 *
 * This function deinitialize SPS endpoints and then
 * deregisters BAM resources from SPS driver.
 *
 * This function should only be called once typically
 * during driver remove.
 *
 * @pce_dev - Pointer to qce_device structure
 *
 */
static void qce_sps_exit(struct qce_device *pce_dev)
{
	qce_sps_exit_ep_conn(pce_dev, &pce_dev->ce_sps.consumer);
	qce_sps_exit_ep_conn(pce_dev, &pce_dev->ce_sps.producer);
	mutex_lock(&bam_register_cnt);
	if ((bam_registry.handle != 0) && (bam_registry.cnt == 1)) {
		sps_deregister_bam_device(pce_dev->ce_sps.bam_handle);
		bam_registry.cnt = 0;
		bam_registry.handle = 0;
	}
	if ((bam_registry.handle != 0) && (bam_registry.cnt > 1))
		bam_registry.cnt--;
	mutex_unlock(&bam_register_cnt);

	iounmap(pce_dev->ce_sps.bam_iobase);
}

static void _aead_sps_producer_callback(struct sps_event_notify *notify)
{
	struct qce_device *pce_dev = (struct qce_device *)
		((struct sps_event_notify *)notify)->user;

	pce_dev->ce_sps.notify = *notify;
	pr_debug("sps ev_id=%d, addr=0x%x, size=0x%x, flags=0x%x\n",
			notify->event_id,
			notify->data.transfer.iovec.addr,
			notify->data.transfer.iovec.size,
			notify->data.transfer.iovec.flags);

	pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_COMP;
	if (pce_dev->ce_sps.consumer_state == QCE_PIPE_STATE_COMP) {
		pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_IDLE;
		pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_IDLE;

		/* done */
		_aead_complete(pce_dev);
	}
};

static void _aead_sps_consumer_callback(struct sps_event_notify *notify)
{
	struct qce_device *pce_dev = (struct qce_device *)
		((struct sps_event_notify *)notify)->user;

	pce_dev->ce_sps.notify = *notify;
	pr_debug("sps ev_id=%d, addr=0x%x, size=0x%x, flags=0x%x\n",
			notify->event_id,
			notify->data.transfer.iovec.addr,
			notify->data.transfer.iovec.size,
			notify->data.transfer.iovec.flags);

	pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_COMP;
	if (pce_dev->ce_sps.producer_state == QCE_PIPE_STATE_COMP) {
		pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_IDLE;
		pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_IDLE;

		/* done */
		_aead_complete(pce_dev);
	}
};

static void _sha_sps_producer_callback(struct sps_event_notify *notify)
{
	struct qce_device *pce_dev = (struct qce_device *)
		((struct sps_event_notify *)notify)->user;

	pce_dev->ce_sps.notify = *notify;
	pr_debug("sps ev_id=%d, addr=0x%x, size=0x%x, flags=0x%x\n",
			notify->event_id,
			notify->data.transfer.iovec.addr,
			notify->data.transfer.iovec.size,
			notify->data.transfer.iovec.flags);

	pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_COMP;
	if (pce_dev->ce_sps.consumer_state == QCE_PIPE_STATE_COMP) {
		pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_IDLE;
		pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_IDLE;

		/* done */
		_sha_complete(pce_dev);
	}
};

static void _sha_sps_consumer_callback(struct sps_event_notify *notify)
{
	struct qce_device *pce_dev = (struct qce_device *)
		((struct sps_event_notify *)notify)->user;

	pce_dev->ce_sps.notify = *notify;
	pr_debug("sps ev_id=%d, addr=0x%x, size=0x%x, flags=0x%x\n",
			notify->event_id,
			notify->data.transfer.iovec.addr,
			notify->data.transfer.iovec.size,
			notify->data.transfer.iovec.flags);

	pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_COMP;
	if (pce_dev->ce_sps.producer_state == QCE_PIPE_STATE_COMP) {
		pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_IDLE;
		pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_IDLE;

		/* done */
	_sha_complete(pce_dev);
	}
};

static void _ablk_cipher_sps_producer_callback(struct sps_event_notify *notify)
{
	struct qce_device *pce_dev = (struct qce_device *)
		((struct sps_event_notify *)notify)->user;

	pce_dev->ce_sps.notify = *notify;
	pr_debug("sps ev_id=%d, addr=0x%x, size=0x%x, flags=0x%x\n",
			notify->event_id,
			notify->data.transfer.iovec.addr,
			notify->data.transfer.iovec.size,
			notify->data.transfer.iovec.flags);

	pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_COMP;
	if (pce_dev->ce_sps.consumer_state == QCE_PIPE_STATE_COMP) {
		pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_IDLE;
		pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_IDLE;

		/* done */
		_ablk_cipher_complete(pce_dev);
	}
};

static void _ablk_cipher_sps_consumer_callback(struct sps_event_notify *notify)
{
	struct qce_device *pce_dev = (struct qce_device *)
		((struct sps_event_notify *)notify)->user;

	pce_dev->ce_sps.notify = *notify;
	pr_debug("sps ev_id=%d, addr=0x%x, size=0x%x, flags=0x%x\n",
			notify->event_id,
			notify->data.transfer.iovec.addr,
			notify->data.transfer.iovec.size,
			notify->data.transfer.iovec.flags);

	pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_COMP;
	if (pce_dev->ce_sps.producer_state == QCE_PIPE_STATE_COMP) {
		pce_dev->ce_sps.consumer_state = QCE_PIPE_STATE_IDLE;
		pce_dev->ce_sps.producer_state = QCE_PIPE_STATE_IDLE;

		/* done */
		_ablk_cipher_complete(pce_dev);
	}
};

static void qce_add_cmd_element(struct qce_device *pdev,
			struct sps_command_element **cmd_ptr, u32 addr,
			u32 data, struct sps_command_element **populate)
{
	(*cmd_ptr)->addr = (uint32_t)(addr + pdev->phy_iobase);
	(*cmd_ptr)->data = data;
	(*cmd_ptr)->mask = 0xFFFFFFFF;
	if (populate != NULL)
		*populate = *cmd_ptr;
	(*cmd_ptr)++ ;
}

static int _setup_cipher_aes_cmdlistptrs(struct qce_device *pdev,
		unsigned char **pvaddr, enum qce_cipher_mode_enum mode,
		bool key_128)
{
	struct sps_command_element *ce_vaddr =
				(struct sps_command_element *)(*pvaddr);
	uint32_t ce_vaddr_start = (uint32_t)(*pvaddr);
	struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_sps.cmdlistptr;
	struct qce_cmdlist_info *pcl_info = NULL;
	int i = 0;
	uint32_t encr_cfg = 0;
	uint32_t key_reg = 0;
	uint32_t xts_key_reg = 0;
	uint32_t iv_reg = 0;
	uint32_t crypto_cfg = 0;
	uint32_t beats = (pdev->ce_sps.ce_burst_size >> 3) - 1;
	uint32_t pipe_pair = pdev->ce_sps.pipe_pair_index;

	crypto_cfg = (beats << CRYPTO_REQ_SIZE) |
			BIT(CRYPTO_MASK_DOUT_INTR) |
			BIT(CRYPTO_MASK_DIN_INTR) |
			BIT(CRYPTO_MASK_OP_DONE_INTR) |
			(0 << CRYPTO_HIGH_SPD_EN_N) |
			(pipe_pair << CRYPTO_PIPE_SET_SELECT);
	/*
	 * Designate chunks of the allocated memory to various
	 * command list pointers related to AES cipher operations defined
	 * in ce_cmdlistptrs_ops structure.
	 */
	switch (mode) {
	case QCE_MODE_CBC:
	case QCE_MODE_CTR:
		if (key_128 == true) {
			cmdlistptr->cipher_aes_128_cbc_ctr.cmdlist =
							(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_aes_128_cbc_ctr);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_AES128 <<
						CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_AES <<
						CRYPTO_ENCR_ALG);
			iv_reg = 4;
			key_reg = 4;
			xts_key_reg = 0;
		} else {
			cmdlistptr->cipher_aes_256_cbc_ctr.cmdlist =
							(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_aes_256_cbc_ctr);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_AES256 <<
							CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_AES <<
							CRYPTO_ENCR_ALG);
			iv_reg = 4;
			key_reg = 8;
			xts_key_reg = 0;
		}
	break;
	case QCE_MODE_ECB:
		if (key_128 == true) {
			cmdlistptr->cipher_aes_128_ecb.cmdlist =
							(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_aes_128_ecb);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_AES128 <<
						CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_AES <<
						CRYPTO_ENCR_ALG) |
					(CRYPTO_ENCR_MODE_ECB <<
						CRYPTO_ENCR_MODE);
			iv_reg = 0;
			key_reg = 4;
			xts_key_reg = 0;
		} else {
			cmdlistptr->cipher_aes_256_ecb.cmdlist =
							(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_aes_256_ecb);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_AES256 <<
							CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_AES <<
							CRYPTO_ENCR_ALG) |
					(CRYPTO_ENCR_MODE_ECB <<
						CRYPTO_ENCR_MODE);
			iv_reg = 0;
			key_reg = 8;
			xts_key_reg = 0;
		}
	break;
	case QCE_MODE_XTS:
		if (key_128 == true) {
			cmdlistptr->cipher_aes_128_xts.cmdlist =
							(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_aes_128_xts);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_AES128 <<
						CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_AES <<
						CRYPTO_ENCR_ALG) |
					(CRYPTO_ENCR_MODE_XTS <<
						CRYPTO_ENCR_MODE);
			iv_reg = 4;
			key_reg = 4;
			xts_key_reg = 4;
		} else {
			cmdlistptr->cipher_aes_256_xts.cmdlist =
							(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_aes_256_xts);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_AES256 <<
							CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_AES <<
							CRYPTO_ENCR_ALG) |
					(CRYPTO_ENCR_MODE_XTS <<
						CRYPTO_ENCR_MODE);
			iv_reg = 4;
			key_reg = 8;
			xts_key_reg = 8;
		}
	break;
	default:
	break;
	}

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, crypto_cfg,
						&pcl_info->crypto_cfg);

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
						&pcl_info->seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg,
						&pcl_info->encr_seg_cfg);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
						&pcl_info->encr_seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
						&pcl_info->encr_seg_start);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG,
				(uint32_t)0xffffffff, &pcl_info->encr_mask);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0,
						&pcl_info->auth_seg_cfg);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0,
						&pcl_info->encr_key);
	for (i = 1; i < key_reg; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
				0, NULL);
	if (xts_key_reg) {
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_XTS_KEY0_REG,
					0, &pcl_info->encr_xts_key);
		for (i = 1; i < xts_key_reg; i++)
			qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
				0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr,
				CRYPTO_ENCR_XTS_DU_SIZE_REG, 0, NULL);
	}
	if (iv_reg) {
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0,
						&pcl_info->encr_cntr_iv);
		for (i = 1; i < iv_reg; i++)
			qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)),
				0, NULL);
	}
	/* Add dummy to  align size to burst-size multiple */
	if (mode == QCE_MODE_XTS) {
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG,
						0, &pcl_info->auth_seg_size);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
						0, &pcl_info->auth_seg_size);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG,
						0, &pcl_info->auth_seg_size);

	}

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
			((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP)),
			&pcl_info->go_proc);

	pcl_info->size = (uint32_t)ce_vaddr - (uint32_t)ce_vaddr_start;
	*pvaddr = (unsigned char *) ce_vaddr;

	return 0;
}

static int _setup_cipher_des_cmdlistptrs(struct qce_device *pdev,
		unsigned char **pvaddr, enum qce_cipher_alg_enum alg,
		bool mode_cbc)
{

	struct sps_command_element *ce_vaddr =
				(struct sps_command_element *)(*pvaddr);
	uint32_t ce_vaddr_start = (uint32_t)(*pvaddr);
	struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_sps.cmdlistptr;
	struct qce_cmdlist_info *pcl_info = NULL;
	int i = 0;
	uint32_t encr_cfg = 0;
	uint32_t key_reg = 0;
	uint32_t iv_reg = 0;
	uint32_t crypto_cfg = 0;
	uint32_t beats = (pdev->ce_sps.ce_burst_size >> 3) - 1;
	uint32_t pipe_pair = pdev->ce_sps.pipe_pair_index;

	crypto_cfg = (beats << CRYPTO_REQ_SIZE) |
			BIT(CRYPTO_MASK_DOUT_INTR) |
			BIT(CRYPTO_MASK_DIN_INTR) |
			BIT(CRYPTO_MASK_OP_DONE_INTR) |
			(0 << CRYPTO_HIGH_SPD_EN_N) |
			(pipe_pair << CRYPTO_PIPE_SET_SELECT);

	/*
	 * Designate chunks of the allocated memory to various
	 * command list pointers related to cipher operations defined
	 * in ce_cmdlistptrs_ops structure.
	 */
	switch (alg) {
	case CIPHER_ALG_DES:
		if (mode_cbc) {
			cmdlistptr->cipher_des_cbc.cmdlist =
						(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_des_cbc);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_DES <<
						CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_DES <<
						CRYPTO_ENCR_ALG) |
					(CRYPTO_ENCR_MODE_CBC <<
							CRYPTO_ENCR_MODE);
			iv_reg = 2;
			key_reg = 2;
		} else {
			cmdlistptr->cipher_des_ecb.cmdlist =
						(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_des_ecb);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_DES <<
						CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_DES <<
						CRYPTO_ENCR_ALG) |
					(CRYPTO_ENCR_MODE_ECB <<
						CRYPTO_ENCR_MODE);
			iv_reg = 0;
			key_reg = 2;
		}
	break;
	case CIPHER_ALG_3DES:
		if (mode_cbc) {
			cmdlistptr->cipher_3des_cbc.cmdlist =
						(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_3des_cbc);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_3DES <<
						CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_DES <<
						CRYPTO_ENCR_ALG) |
					(CRYPTO_ENCR_MODE_CBC <<
							CRYPTO_ENCR_MODE);
			iv_reg = 2;
			key_reg = 6;
		} else {
			cmdlistptr->cipher_3des_ecb.cmdlist =
						(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->cipher_3des_ecb);

			encr_cfg = (CRYPTO_ENCR_KEY_SZ_3DES <<
						CRYPTO_ENCR_KEY_SZ) |
					(CRYPTO_ENCR_ALG_DES <<
						CRYPTO_ENCR_ALG) |
					(CRYPTO_ENCR_MODE_ECB <<
						CRYPTO_ENCR_MODE);
			iv_reg = 0;
			key_reg = 6;
		}
	break;
	default:
	break;
	}

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG, crypto_cfg,
						&pcl_info->crypto_cfg);

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
						&pcl_info->seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg,
						&pcl_info->encr_seg_cfg);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
						&pcl_info->encr_seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
						&pcl_info->encr_seg_start);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0,
						&pcl_info->auth_seg_cfg);

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0,
						&pcl_info->encr_key);
	for (i = 1; i < key_reg; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
				0, NULL);
	if (iv_reg) {
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0,
						&pcl_info->encr_cntr_iv);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR1_IV1_REG, 0,
								NULL);
		/* Add 2 dummy to  align size to burst-size multiple */
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR2_IV2_REG, 0,
								NULL);
	}
	/* Add dummy to  align size to burst-size multiple */
	if (!mode_cbc) {
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG,
						0, &pcl_info->auth_seg_size);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
						0, &pcl_info->auth_seg_size);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG,
						0, &pcl_info->auth_seg_size);

	}

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
			((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP)),
			&pcl_info->go_proc);

	pcl_info->size = (uint32_t)ce_vaddr - (uint32_t)ce_vaddr_start;
	*pvaddr = (unsigned char *) ce_vaddr;

	return 0;
}

static int _setup_auth_cmdlistptrs(struct qce_device *pdev,
		unsigned char **pvaddr, enum qce_hash_alg_enum alg,
		bool key_128)
{
	struct sps_command_element *ce_vaddr =
			(struct sps_command_element *)(*pvaddr);
	uint32_t ce_vaddr_start = (uint32_t)(*pvaddr);
	struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_sps.cmdlistptr;
	struct qce_cmdlist_info *pcl_info = NULL;
	int i = 0;
	uint32_t key_reg = 0;
	uint32_t auth_cfg = 0;
	uint32_t iv_reg = 0;
	uint32_t crypto_cfg = 0;
	uint32_t beats = (pdev->ce_sps.ce_burst_size >> 3) - 1;
	uint32_t pipe_pair = pdev->ce_sps.pipe_pair_index;

	crypto_cfg = (beats << CRYPTO_REQ_SIZE) |
			BIT(CRYPTO_MASK_DOUT_INTR) |
			BIT(CRYPTO_MASK_DIN_INTR) |
			BIT(CRYPTO_MASK_OP_DONE_INTR) |
			(0 << CRYPTO_HIGH_SPD_EN_N) |
			(pipe_pair << CRYPTO_PIPE_SET_SELECT);
	/*
	 * Designate chunks of the allocated memory to various
	 * command list pointers related to authentication operations
	 * defined in ce_cmdlistptrs_ops structure.
	 */
	switch (alg) {
	case QCE_HASH_SHA1:
		cmdlistptr->auth_sha1.cmdlist = (uint32_t)ce_vaddr;
		pcl_info = &(cmdlistptr->auth_sha1);

		auth_cfg = (CRYPTO_AUTH_MODE_HASH << CRYPTO_AUTH_MODE)|
				(CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) |
				(CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
				(CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
		iv_reg = 5;
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					crypto_cfg, &pcl_info->crypto_cfg);
		/* 1 dummy write */
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
							0, NULL);

	break;
	case QCE_HASH_SHA256:
		cmdlistptr->auth_sha256.cmdlist = (uint32_t)ce_vaddr;
		pcl_info = &(cmdlistptr->auth_sha256);

		auth_cfg = (CRYPTO_AUTH_MODE_HASH << CRYPTO_AUTH_MODE)|
				(CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE) |
				(CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
				(CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
		iv_reg = 8;
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					crypto_cfg, &pcl_info->crypto_cfg);
		/* 2 dummy writes */
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
								0, NULL);
	break;
	case QCE_HASH_SHA1_HMAC:
		cmdlistptr->auth_sha1_hmac.cmdlist = (uint32_t)ce_vaddr;
		pcl_info = &(cmdlistptr->auth_sha1_hmac);

		auth_cfg = (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)|
				(CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) |
				(CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
				(CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
		key_reg = 16;
		iv_reg = 5;
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					crypto_cfg, &pcl_info->crypto_cfg);
		/* 1 dummy write */
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
							0, NULL);
	break;
	case QCE_AEAD_SHA1_HMAC:
		cmdlistptr->aead_sha1_hmac.cmdlist = (uint32_t)ce_vaddr;
		pcl_info = &(cmdlistptr->aead_sha1_hmac);

		auth_cfg = (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)|
				(CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) |
				(CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
				(CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS) |
				(1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST);

		key_reg = 16;
		iv_reg = 5;
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					crypto_cfg, &pcl_info->crypto_cfg);
		/* 2 dummy writes */
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
							0, NULL);
	break;
	case QCE_HASH_SHA256_HMAC:
		cmdlistptr->auth_sha256_hmac.cmdlist = (uint32_t)ce_vaddr;
		pcl_info = &(cmdlistptr->auth_sha256_hmac);

		auth_cfg = (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)|
				(CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE) |
				(CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
				(CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
		key_reg = 16;
		iv_reg = 8;

		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					crypto_cfg, &pcl_info->crypto_cfg);
		/* 2 dummy writes */
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
								0, NULL);
	break;
	case QCE_HASH_AES_CMAC:
		if (key_128 == true) {
			cmdlistptr->auth_aes_128_cmac.cmdlist =
						(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->auth_aes_128_cmac);

			auth_cfg = (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) |
				(CRYPTO_AUTH_MODE_CMAC << CRYPTO_AUTH_MODE)|
				(CRYPTO_AUTH_SIZE_ENUM_16_BYTES <<
							CRYPTO_AUTH_SIZE) |
				(CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) |
				(CRYPTO_AUTH_KEY_SZ_AES128 <<
							CRYPTO_AUTH_KEY_SIZE) |
				(CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
			key_reg = 4;
		} else {
			cmdlistptr->auth_aes_256_cmac.cmdlist =
							(uint32_t)ce_vaddr;
			pcl_info = &(cmdlistptr->auth_aes_256_cmac);

			auth_cfg = (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST)|
				(CRYPTO_AUTH_MODE_CMAC << CRYPTO_AUTH_MODE)|
				(CRYPTO_AUTH_SIZE_ENUM_16_BYTES <<
							CRYPTO_AUTH_SIZE) |
				(CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) |
				(CRYPTO_AUTH_KEY_SZ_AES256 <<
							CRYPTO_AUTH_KEY_SIZE) |
				(CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
			key_reg = 8;
		}
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					crypto_cfg, &pcl_info->crypto_cfg);
		/* 2 dummy writes */
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
								0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
								0, NULL);
	break;
	default:
	break;
	}

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
						&pcl_info->seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, 0,
						&pcl_info->encr_seg_cfg);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
					auth_cfg, &pcl_info->auth_seg_cfg);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0,
						&pcl_info->auth_seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0,
						&pcl_info->auth_seg_start);

	if (alg == QCE_HASH_AES_CMAC) {
		/* reset auth iv, bytecount and key  registers */
		for (i = 0; i < 16; i++)
			qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t)),
				0, NULL);
		for (i = 0; i < 16; i++)
			qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)),
				0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG,
						0, NULL);
	} else {
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_IV0_REG, 0,
							&pcl_info->auth_iv);
		for (i = 1; i < iv_reg; i++)
			qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)),
				0, NULL);
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG,
						0, &pcl_info->auth_bytecount);
	}
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG, 0, NULL);

	if (key_reg) {
		qce_add_cmd_element(pdev, &ce_vaddr,
				CRYPTO_AUTH_KEY0_REG, 0, &pcl_info->auth_key);
		for (i = 1; i < key_reg; i++)
			qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)),
				0, NULL);
	}
	if (alg != QCE_AEAD_SHA1_HMAC)
		qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
			((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP)),
			&pcl_info->go_proc);

	pcl_info->size = (uint32_t)ce_vaddr - (uint32_t)ce_vaddr_start;
	*pvaddr = (unsigned char *) ce_vaddr;

	return 0;
}

static int _setup_aead_cmdlistptrs(struct qce_device *pdev,
				unsigned char **pvaddr, bool key_128)
{
	struct sps_command_element *ce_vaddr =
				(struct sps_command_element *)(*pvaddr);
	uint32_t ce_vaddr_start = (uint32_t)(*pvaddr);
	struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_sps.cmdlistptr;
	struct qce_cmdlist_info *pcl_info = NULL;
	int i = 0;
	uint32_t encr_cfg = 0;
	uint32_t auth_cfg = 0;
	uint32_t key_reg = 0;
	uint32_t crypto_cfg = 0;
	uint32_t beats = (pdev->ce_sps.ce_burst_size >> 3) - 1;
	uint32_t pipe_pair = pdev->ce_sps.pipe_pair_index;

	crypto_cfg = (beats << CRYPTO_REQ_SIZE) |
			BIT(CRYPTO_MASK_DOUT_INTR) |
			BIT(CRYPTO_MASK_DIN_INTR) |
			BIT(CRYPTO_MASK_OP_DONE_INTR) |
			(0 << CRYPTO_HIGH_SPD_EN_N) |
			(pipe_pair << CRYPTO_PIPE_SET_SELECT);
	/*
	 * Designate chunks of the allocated memory to various
	 * command list pointers related to aead operations
	 * defined in ce_cmdlistptrs_ops structure.
	 */
	if (key_128 == true) {
		cmdlistptr->aead_aes_128_ccm.cmdlist = (uint32_t)ce_vaddr;
		pcl_info = &(cmdlistptr->aead_aes_128_ccm);

		auth_cfg = (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) |
			(CRYPTO_AUTH_MODE_CCM << CRYPTO_AUTH_MODE)|
			(CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) |
			(CRYPTO_AUTH_KEY_SZ_AES128 << CRYPTO_AUTH_KEY_SIZE);
		auth_cfg &= ~(1 << CRYPTO_USE_HW_KEY_AUTH);
		encr_cfg = (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) |
			(CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
			((CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE));
		key_reg = 4;
	} else {

		cmdlistptr->aead_aes_256_ccm.cmdlist = (uint32_t)ce_vaddr;
		pcl_info = &(cmdlistptr->aead_aes_256_ccm);

		auth_cfg = (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) |
			(CRYPTO_AUTH_MODE_CCM << CRYPTO_AUTH_MODE)|
			(CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) |
			(CRYPTO_AUTH_KEY_SZ_AES256 << CRYPTO_AUTH_KEY_SIZE) |
			((MAX_NONCE/sizeof(uint32_t)) <<
						CRYPTO_AUTH_NONCE_NUM_WORDS);
		auth_cfg &= ~(1 << CRYPTO_USE_HW_KEY_AUTH);
		encr_cfg = (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) |
			(CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
			((CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE));
		key_reg = 8;
	}
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
				crypto_cfg, &pcl_info->crypto_cfg);

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, 0, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
									NULL);
	/* add 1 dummy */
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
						&pcl_info->seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG,
					encr_cfg, &pcl_info->encr_seg_cfg);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
						&pcl_info->encr_seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
						&pcl_info->encr_seg_start);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG,
				(uint32_t)0xffffffff, &pcl_info->encr_mask);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
					auth_cfg, &pcl_info->auth_seg_cfg);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0,
						&pcl_info->auth_seg_size);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0,
						&pcl_info->auth_seg_start);
	/* reset auth iv, bytecount and key  registers */
	for (i = 0; i < 8; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t)),
				0, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG,
					0, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG,
					0, NULL);
	for (i = 0; i < 16; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_AUTH_KEY0_REG + i * sizeof(uint32_t)),
				0, NULL);
	/* set auth key */
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_KEY0_REG, 0,
							&pcl_info->auth_key);
	for (i = 1; i < key_reg; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_AUTH_KEY0_REG + i * sizeof(uint32_t)),
				0, NULL);
	/* set NONCE info */
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_INFO_NONCE0_REG, 0,
						&pcl_info->auth_nonce_info);
	for (i = 1; i < 4; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_AUTH_INFO_NONCE0_REG +
				i * sizeof(uint32_t)), 0, NULL);

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0,
						&pcl_info->encr_key);
	for (i = 1; i < key_reg; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
				0, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0,
						&pcl_info->encr_cntr_iv);
	for (i = 1; i < 4; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
				(CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)),
				0, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_CCM_INT_CNTR0_REG, 0,
						&pcl_info->encr_ccm_cntr_iv);
	for (i = 1; i < 4; i++)
		qce_add_cmd_element(pdev, &ce_vaddr,
			(CRYPTO_ENCR_CCM_INT_CNTR0_REG + i * sizeof(uint32_t)),
			0, NULL);

	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
			((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP)),
			&pcl_info->go_proc);

	pcl_info->size = (uint32_t)ce_vaddr - (uint32_t)ce_vaddr_start;
	*pvaddr = (unsigned char *) ce_vaddr;

	return 0;
}

static int _setup_unlock_pipe_cmdlistptrs(struct qce_device *pdev,
		unsigned char **pvaddr)
{
	struct sps_command_element *ce_vaddr =
			(struct sps_command_element *)(*pvaddr);
	uint32_t ce_vaddr_start = (uint32_t)(*pvaddr);
	struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_sps.cmdlistptr;
	struct qce_cmdlist_info *pcl_info = NULL;

	cmdlistptr->unlock_all_pipes.cmdlist = (uint32_t)ce_vaddr;
	pcl_info = &(cmdlistptr->unlock_all_pipes);

	/*
	 * Designate chunks of the allocated memory to command list
	 * to unlock pipes.
	 */
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					CRYPTO_CONFIG_RESET, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					CRYPTO_CONFIG_RESET, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					CRYPTO_CONFIG_RESET, NULL);
	qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
					CRYPTO_CONFIG_RESET, NULL);
	pcl_info->size = (uint32_t)ce_vaddr - (uint32_t)ce_vaddr_start;
	*pvaddr = (unsigned char *) ce_vaddr;

	return 0;
}

static int qce_setup_cmdlistptrs(struct qce_device *pdev,
					unsigned char **pvaddr)
{
	struct sps_command_element *ce_vaddr =
				(struct sps_command_element *)(*pvaddr);
	/*
	 * Designate chunks of the allocated memory to various
	 * command list pointers related to operations defined
	 * in ce_cmdlistptrs_ops structure.
	 */
	ce_vaddr =
		(struct sps_command_element *) ALIGN(((unsigned int) ce_vaddr),
									16);
	*pvaddr = (unsigned char *) ce_vaddr;

	_setup_cipher_aes_cmdlistptrs(pdev, pvaddr, QCE_MODE_CBC, true);
	_setup_cipher_aes_cmdlistptrs(pdev, pvaddr, QCE_MODE_CTR, true);
	_setup_cipher_aes_cmdlistptrs(pdev, pvaddr, QCE_MODE_ECB, true);
	_setup_cipher_aes_cmdlistptrs(pdev, pvaddr, QCE_MODE_XTS, true);
	_setup_cipher_aes_cmdlistptrs(pdev, pvaddr, QCE_MODE_CBC, false);
	_setup_cipher_aes_cmdlistptrs(pdev, pvaddr, QCE_MODE_CTR, false);
	_setup_cipher_aes_cmdlistptrs(pdev, pvaddr, QCE_MODE_ECB, false);
	_setup_cipher_aes_cmdlistptrs(pdev, pvaddr, QCE_MODE_XTS, false);

	_setup_cipher_des_cmdlistptrs(pdev, pvaddr, CIPHER_ALG_DES, true);
	_setup_cipher_des_cmdlistptrs(pdev, pvaddr, CIPHER_ALG_DES, false);
	_setup_cipher_des_cmdlistptrs(pdev, pvaddr, CIPHER_ALG_3DES, true);
	_setup_cipher_des_cmdlistptrs(pdev, pvaddr, CIPHER_ALG_3DES, false);

	_setup_auth_cmdlistptrs(pdev, pvaddr, QCE_HASH_SHA1, false);
	_setup_auth_cmdlistptrs(pdev, pvaddr, QCE_HASH_SHA256, false);

	_setup_auth_cmdlistptrs(pdev, pvaddr, QCE_HASH_SHA1_HMAC, false);
	_setup_auth_cmdlistptrs(pdev, pvaddr, QCE_HASH_SHA256_HMAC, false);

	_setup_auth_cmdlistptrs(pdev, pvaddr, QCE_HASH_AES_CMAC, true);
	_setup_auth_cmdlistptrs(pdev, pvaddr, QCE_HASH_AES_CMAC, false);

	_setup_auth_cmdlistptrs(pdev, pvaddr, QCE_AEAD_SHA1_HMAC, false);

	_setup_aead_cmdlistptrs(pdev, pvaddr, true);
	_setup_aead_cmdlistptrs(pdev, pvaddr, false);
	_setup_unlock_pipe_cmdlistptrs(pdev, pvaddr);

	return 0;
}

static int qce_setup_ce_sps_data(struct qce_device *pce_dev)
{
	unsigned char *vaddr;

	vaddr = pce_dev->coh_vmem;
	vaddr = (unsigned char *) ALIGN(((unsigned int)vaddr),  16);

	/* Allow for 256 descriptor (cmd and data) entries per pipe */
	pce_dev->ce_sps.in_transfer.iovec = (struct sps_iovec *)vaddr;
	pce_dev->ce_sps.in_transfer.iovec_phys =
					(uint32_t)GET_PHYS_ADDR(vaddr);
	vaddr += MAX_BAM_DESCRIPTORS * 8;

	pce_dev->ce_sps.out_transfer.iovec = (struct sps_iovec *)vaddr;
	pce_dev->ce_sps.out_transfer.iovec_phys =
					(uint32_t)GET_PHYS_ADDR(vaddr);
	vaddr += MAX_BAM_DESCRIPTORS * 8;

	qce_setup_cmdlistptrs(pce_dev, &vaddr);
	pce_dev->ce_sps.result_dump = (uint32_t)vaddr;
	pce_dev->ce_sps.result = (struct ce_result_dump_format *)vaddr;
	vaddr += 128;

	return 0;
}

int qce_aead_sha1_hmac_setup(struct qce_req *creq, struct crypto_aead *aead,
				struct qce_cmdlist_info *cmdlistinfo)
{
	uint32_t authk_size_in_word = creq->authklen/sizeof(uint32_t);
	uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = {
			0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
	struct sps_command_element *pce = NULL;
	struct aead_request *areq = (struct aead_request *)creq->areq;
	int i;

	_byte_stream_to_net_words(mackey32, creq->authkey,
					creq->authklen);
	pce = cmdlistinfo->auth_key;
	for (i = 0; i < authk_size_in_word; i++, pce++)
		pce->data = mackey32[i];
	pce = cmdlistinfo->auth_iv;
	for (i = 0; i < 5; i++, pce++)
		pce->data = _std_init_vector_sha1[i];
	/* write auth seg size */
	pce = cmdlistinfo->auth_seg_size;
	pce->data = creq->cryptlen + areq->assoclen + crypto_aead_ivsize(aead);

	/* write auth seg size start*/
	pce = cmdlistinfo->auth_seg_start;
	pce->data = 0;

	return 0;
}

int qce_aead_req(void *handle, struct qce_req *q_req)
{
	struct qce_device *pce_dev = (struct qce_device *) handle;
	struct aead_request *areq = (struct aead_request *) q_req->areq;
	uint32_t authsize = q_req->authsize;
	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	uint32_t ivsize = 0;
	uint32_t totallen_in, out_len;
	uint32_t hw_pad_out = 0;
	int rc = 0;
	int ce_burst_size;
	struct qce_cmdlist_info *cmdlistinfo = NULL;
	struct qce_cmdlist_info *auth_cmdlistinfo = NULL;

	if (q_req->mode != QCE_MODE_CCM)
		ivsize = crypto_aead_ivsize(aead);

	ce_burst_size = pce_dev->ce_sps.ce_burst_size;
	if (q_req->dir == QCE_ENCRYPT) {
		q_req->cryptlen = areq->cryptlen;
			totallen_in = q_req->cryptlen + areq->assoclen + ivsize;
		if (q_req->mode == QCE_MODE_CCM) {
			out_len = areq->cryptlen + authsize;
			hw_pad_out = ALIGN(authsize, ce_burst_size) - authsize;
		} else {
			out_len = areq->cryptlen;
		}
	} else {
		q_req->cryptlen = areq->cryptlen - authsize;
		if (q_req->mode == QCE_MODE_CCM)
			totallen_in = areq->cryptlen + areq->assoclen;
		else
			totallen_in = q_req->cryptlen + areq->assoclen + ivsize;
		out_len = q_req->cryptlen;
		hw_pad_out = authsize;
	}

	pce_dev->assoc_nents = count_sg(areq->assoc, areq->assoclen);
	pce_dev->src_nents = count_sg(areq->src, areq->cryptlen);
	pce_dev->ivsize = q_req->ivsize;
	pce_dev->authsize = q_req->authsize;
	pce_dev->phy_iv_in = 0;

	/* associated data input */
	dma_map_sg(pce_dev->pdev, areq->assoc, pce_dev->assoc_nents,
					 DMA_TO_DEVICE);
	/* cipher input */
	dma_map_sg(pce_dev->pdev, areq->src, pce_dev->src_nents,
			(areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
							DMA_TO_DEVICE);
	/* cipher + mac output  for encryption    */
	if (areq->src != areq->dst) {
		pce_dev->dst_nents = count_sg(areq->dst, out_len);
		dma_map_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents,
				DMA_FROM_DEVICE);
	} else {
		pce_dev->dst_nents = pce_dev->src_nents;
	}

	_ce_get_cipher_cmdlistinfo(pce_dev, q_req, &cmdlistinfo);
	/* set up crypto device */
	rc = _ce_setup_cipher(pce_dev, q_req, totallen_in,
				areq->assoclen + ivsize, cmdlistinfo);
	if (rc < 0)
		goto bad;

	if (q_req->mode != QCE_MODE_CCM) {
		rc = qce_aead_sha1_hmac_setup(q_req, aead, auth_cmdlistinfo);
		if (rc < 0)
			goto bad;
		/* overwrite seg size */
		cmdlistinfo->seg_size->data = totallen_in;
		/* cipher iv for input */
		pce_dev->phy_iv_in = dma_map_single(pce_dev->pdev, q_req->iv,
			ivsize, DMA_TO_DEVICE);
	}

	/* setup for callback, and issue command to bam */
	pce_dev->areq = q_req->areq;
	pce_dev->qce_cb = q_req->qce_cb;

	/* Register callback event for EOT (End of transfer) event. */
	pce_dev->ce_sps.producer.event.callback = _aead_sps_producer_callback;
	rc = sps_register_event(pce_dev->ce_sps.producer.pipe,
					&pce_dev->ce_sps.producer.event);
	if (rc) {
		pr_err("Producer callback registration failed rc = %d\n", rc);
		goto bad;
	}

	/* Register callback event for EOT (End of transfer) event. */
	pce_dev->ce_sps.consumer.event.callback = _aead_sps_consumer_callback;
	rc = sps_register_event(pce_dev->ce_sps.consumer.pipe,
					&pce_dev->ce_sps.consumer.event);
	if (rc) {
		pr_err("Consumer callback registration failed rc = %d\n", rc);
		goto bad;
	}

	_qce_sps_iovec_count_init(pce_dev);

	_qce_sps_add_cmd(pce_dev, 0, cmdlistinfo,
					&pce_dev->ce_sps.in_transfer);

	if (pce_dev->ce_sps.minor_version == 0) {
		_qce_sps_add_sg_data(pce_dev, areq->src, totallen_in,
					&pce_dev->ce_sps.in_transfer);

		_qce_set_eot_flag(&pce_dev->ce_sps.in_transfer);
		_qce_sps_add_sg_data(pce_dev, areq->dst, out_len +
					areq->assoclen + hw_pad_out,
				&pce_dev->ce_sps.out_transfer);
		_qce_sps_add_data(GET_PHYS_ADDR(pce_dev->ce_sps.result_dump),
					CRYPTO_RESULT_DUMP_SIZE,
					&pce_dev->ce_sps.out_transfer);
	} else {
		_qce_sps_add_sg_data(pce_dev, areq->assoc, areq->assoclen,
					 &pce_dev->ce_sps.in_transfer);
		_qce_sps_add_data((uint32_t)pce_dev->phy_iv_in, ivsize,
					&pce_dev->ce_sps.in_transfer);
		_qce_sps_add_sg_data(pce_dev, areq->src, areq->cryptlen,
					&pce_dev->ce_sps.in_transfer);
		_qce_set_eot_flag(&pce_dev->ce_sps.in_transfer);

		/* Pass through to ignore associated (+iv, if applicable) data*/
		_qce_sps_add_data(GET_PHYS_ADDR(pce_dev->ce_sps.ignore_buffer),
				(ivsize + areq->assoclen),
				&pce_dev->ce_sps.out_transfer);
		_qce_sps_add_sg_data(pce_dev, areq->dst, out_len,
					&pce_dev->ce_sps.out_transfer);
		/* Pass through to ignore hw_pad (padding of the MAC data) */
		_qce_sps_add_data(GET_PHYS_ADDR(pce_dev->ce_sps.ignore_buffer),
				hw_pad_out, &pce_dev->ce_sps.out_transfer);

		_qce_sps_add_data(GET_PHYS_ADDR(pce_dev->ce_sps.result_dump),
			CRYPTO_RESULT_DUMP_SIZE, &pce_dev->ce_sps.out_transfer);
	}
	rc = _qce_sps_transfer(pce_dev);
	if (rc)
		goto bad;
	return 0;

bad:
	if (pce_dev->assoc_nents) {
		dma_unmap_sg(pce_dev->pdev, areq->assoc, pce_dev->assoc_nents,
				DMA_TO_DEVICE);
	}
	if (pce_dev->src_nents) {
		dma_unmap_sg(pce_dev->pdev, areq->src, pce_dev->src_nents,
				(areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
								DMA_TO_DEVICE);
	}
	if (areq->src != areq->dst) {
		dma_unmap_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents,
				DMA_FROM_DEVICE);
	}
	if (pce_dev->phy_iv_in) {
		dma_unmap_single(pce_dev->pdev, pce_dev->phy_iv_in,
				ivsize, DMA_TO_DEVICE);
	}

	return rc;
}
EXPORT_SYMBOL(qce_aead_req);

int qce_ablk_cipher_req(void *handle, struct qce_req *c_req)
{
	int rc = 0;
	struct qce_device *pce_dev = (struct qce_device *) handle;
	struct ablkcipher_request *areq = (struct ablkcipher_request *)
						c_req->areq;
	struct qce_cmdlist_info *cmdlistinfo = NULL;

	pce_dev->src_nents = 0;
	pce_dev->dst_nents = 0;
	_ce_get_cipher_cmdlistinfo(pce_dev, c_req, &cmdlistinfo);

	/* cipher input */
	pce_dev->src_nents = count_sg(areq->src, areq->nbytes);

	dma_map_sg(pce_dev->pdev, areq->src, pce_dev->src_nents,
		(areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
							DMA_TO_DEVICE);
	/* cipher output */
	if (areq->src != areq->dst) {
		pce_dev->dst_nents = count_sg(areq->dst, areq->nbytes);
			dma_map_sg(pce_dev->pdev, areq->dst, pce_dev->dst_nents,
							DMA_FROM_DEVICE);
	} else {
		pce_dev->dst_nents = pce_dev->src_nents;
	}
	/* set up crypto device */
	rc = _ce_setup_cipher(pce_dev, c_req, areq->nbytes, 0, cmdlistinfo);
	if (rc < 0)
		goto bad;

	/* setup for client callback, and issue command to BAM */
	pce_dev->areq = areq;
	pce_dev->qce_cb = c_req->qce_cb;

	/* Register callback event for EOT (End of transfer) event. */
	pce_dev->ce_sps.producer.event.callback =
				_ablk_cipher_sps_producer_callback;
	rc = sps_register_event(pce_dev->ce_sps.producer.pipe,
					&pce_dev->ce_sps.producer.event);
	if (rc) {
		pr_err("Producer callback registration failed rc = %d\n", rc);
		goto bad;
	}

	/* Register callback event for EOT (End of transfer) event. */
	pce_dev->ce_sps.consumer.event.callback =
			_ablk_cipher_sps_consumer_callback;
	rc = sps_register_event(pce_dev->ce_sps.consumer.pipe,
					&pce_dev->ce_sps.consumer.event);
	if (rc) {
		pr_err("Consumer callback registration failed rc = %d\n", rc);
		goto bad;
	}

	_qce_sps_iovec_count_init(pce_dev);

	_qce_sps_add_cmd(pce_dev, 0, cmdlistinfo,
					&pce_dev->ce_sps.in_transfer);
	_qce_sps_add_sg_data(pce_dev, areq->src, areq->nbytes,
					&pce_dev->ce_sps.in_transfer);
	_qce_set_eot_flag(&pce_dev->ce_sps.in_transfer);

	_qce_sps_add_sg_data(pce_dev, areq->dst, areq->nbytes,
					&pce_dev->ce_sps.out_transfer);
	_qce_sps_add_data(GET_PHYS_ADDR(pce_dev->ce_sps.result_dump),
					CRYPTO_RESULT_DUMP_SIZE,
					  &pce_dev->ce_sps.out_transfer);
	rc = _qce_sps_transfer(pce_dev);
	if (rc)
		goto bad;
		return 0;
bad:
	if (pce_dev->dst_nents) {
		dma_unmap_sg(pce_dev->pdev, areq->dst,
		pce_dev->dst_nents, DMA_FROM_DEVICE);
	}
	if (pce_dev->src_nents) {
		dma_unmap_sg(pce_dev->pdev, areq->src,
				pce_dev->src_nents,
				(areq->src == areq->dst) ?
				DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
	}
	return rc;
}
EXPORT_SYMBOL(qce_ablk_cipher_req);

int qce_process_sha_req(void *handle, struct qce_sha_req *sreq)
{
	struct qce_device *pce_dev = (struct qce_device *) handle;
	int rc;

	struct ahash_request *areq = (struct ahash_request *)sreq->areq;
	struct qce_cmdlist_info *cmdlistinfo = NULL;

	pce_dev->src_nents = count_sg(sreq->src, sreq->size);
	_ce_get_hash_cmdlistinfo(pce_dev, sreq, &cmdlistinfo);
	dma_map_sg(pce_dev->pdev, sreq->src, pce_dev->src_nents,
							DMA_TO_DEVICE);
	rc = _ce_setup_hash(pce_dev, sreq, cmdlistinfo);
	if (rc < 0)
		goto bad;

	pce_dev->areq = areq;
	pce_dev->qce_cb = sreq->qce_cb;

	/* Register callback event for EOT (End of transfer) event. */
	pce_dev->ce_sps.producer.event.callback = _sha_sps_producer_callback;
	rc = sps_register_event(pce_dev->ce_sps.producer.pipe,
					&pce_dev->ce_sps.producer.event);
	if (rc) {
		pr_err("Producer callback registration failed rc = %d\n", rc);
		goto bad;
	}

	/* Register callback event for EOT (End of transfer) event. */
	pce_dev->ce_sps.consumer.event.callback = _sha_sps_consumer_callback;
	rc = sps_register_event(pce_dev->ce_sps.consumer.pipe,
					&pce_dev->ce_sps.consumer.event);
	if (rc) {
		pr_err("Consumer callback registration failed rc = %d\n", rc);
		goto bad;
	}

	_qce_sps_iovec_count_init(pce_dev);

	_qce_sps_add_cmd(pce_dev, 0, cmdlistinfo,
					&pce_dev->ce_sps.in_transfer);
	_qce_sps_add_sg_data(pce_dev, areq->src, areq->nbytes,
						 &pce_dev->ce_sps.in_transfer);
	_qce_set_eot_flag(&pce_dev->ce_sps.in_transfer);

	_qce_sps_add_data(GET_PHYS_ADDR(pce_dev->ce_sps.result_dump),
					CRYPTO_RESULT_DUMP_SIZE,
					  &pce_dev->ce_sps.out_transfer);
	rc = _qce_sps_transfer(pce_dev);
	if (rc)
		goto bad;
		return 0;
bad:
	if (pce_dev->src_nents) {
		dma_unmap_sg(pce_dev->pdev, sreq->src,
				pce_dev->src_nents, DMA_TO_DEVICE);
	}
	return rc;
}
EXPORT_SYMBOL(qce_process_sha_req);

static int __qce_get_device_tree_data(struct platform_device *pdev,
		struct qce_device *pce_dev)
{
	struct resource *resource;
	int rc = 0;

	if (of_property_read_u32((&pdev->dev)->of_node,
				"qcom,bam-pipe-pair",
				&pce_dev->ce_sps.pipe_pair_index)) {
		pr_err("Fail to get bam pipe pair information.\n");
		return -EINVAL;
	} else {
		pr_warn("bam_pipe_pair=0x%x", pce_dev->ce_sps.pipe_pair_index);
	}
	pce_dev->ce_sps.dest_pipe_index	= 2 * pce_dev->ce_sps.pipe_pair_index;
	pce_dev->ce_sps.src_pipe_index	= pce_dev->ce_sps.dest_pipe_index + 1;

	resource = platform_get_resource_byname(pdev, IORESOURCE_MEM,
							"crypto-base");
	if (resource) {
		pce_dev->phy_iobase = resource->start;
		pce_dev->iobase = ioremap_nocache(resource->start,
					resource_size(resource));
		if (!pce_dev->iobase) {
			pr_err("Can not map CRYPTO io memory\n");
			return -ENOMEM;
		}
	} else {
		pr_err("CRYPTO HW mem unavailable.\n");
		return -ENODEV;
	}
	pr_warn("ce_phy_reg_base=0x%x  ", pce_dev->phy_iobase);
	pr_warn("ce_virt_reg_base=0x%x\n", (uint32_t)pce_dev->iobase);

	resource = platform_get_resource_byname(pdev, IORESOURCE_MEM,
							"crypto-bam-base");
	if (resource) {
		pce_dev->ce_sps.bam_mem = resource->start;
		pce_dev->ce_sps.bam_iobase = ioremap_nocache(resource->start,
					resource_size(resource));
		if (!pce_dev->iobase) {
			rc = -ENOMEM;
			pr_err("Can not map BAM io memory\n");
			goto err_getting_bam_info;
		}
	} else {
		pr_err("CRYPTO BAM mem unavailable.\n");
		rc = -ENODEV;
		goto err_getting_bam_info;
	}
	pr_warn("ce_bam_phy_reg_base=0x%x  ", pce_dev->ce_sps.bam_mem);
	pr_warn("ce_bam_virt_reg_base=0x%x\n",
				(uint32_t)pce_dev->ce_sps.bam_iobase);

	resource  = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
	if (resource) {
		pce_dev->ce_sps.bam_irq = resource->start;
		pr_warn("CRYPTO BAM IRQ = %d.\n", pce_dev->ce_sps.bam_irq);
	} else {
		pr_err("CRYPTO BAM IRQ unavailable.\n");
		goto err_dev;
	}
	return rc;
err_dev:
	if (pce_dev->ce_sps.bam_iobase)
		iounmap(pce_dev->ce_sps.bam_iobase);

err_getting_bam_info:
	if (pce_dev->iobase)
		iounmap(pce_dev->iobase);

	return rc;
}

static int __qce_init_clk(struct qce_device *pce_dev)
{
	int rc = 0;
	struct clk *ce_core_clk;
	struct clk *ce_clk;
	struct clk *ce_core_src_clk;

	/* Get CE3 src core clk. */
	ce_core_src_clk = clk_get(pce_dev->pdev, "core_clk_src");
	if (!IS_ERR(ce_core_src_clk)) {
		pce_dev->ce_core_src_clk = ce_core_src_clk;

		/* Set the core src clk @100Mhz */
		rc = clk_set_rate(pce_dev->ce_core_src_clk, 100000000);
		if (rc) {
			clk_put(pce_dev->ce_core_src_clk);
			pr_err("Unable to set the core src clk @100Mhz.\n");
			goto err_clk;
		}
	} else {
		pr_warn("Unable to get CE core src clk, set to NULL\n");
		pce_dev->ce_core_src_clk = NULL;
	}

	/* Get CE core clk */
	ce_core_clk = clk_get(pce_dev->pdev, "core_clk");
	if (IS_ERR(ce_core_clk)) {
		rc = PTR_ERR(ce_core_clk);
		pr_err("Unable to get CE core clk\n");
		if (pce_dev->ce_core_src_clk != NULL)
			clk_put(pce_dev->ce_core_src_clk);
		goto err_clk;
	}
	pce_dev->ce_core_clk = ce_core_clk;

	/* Get CE Interface clk */
	ce_clk = clk_get(pce_dev->pdev, "iface_clk");
	if (IS_ERR(ce_clk)) {
		rc = PTR_ERR(ce_clk);
		pr_err("Unable to get CE interface clk\n");
		if (pce_dev->ce_core_src_clk != NULL)
			clk_put(pce_dev->ce_core_src_clk);
		clk_put(pce_dev->ce_core_clk);
		goto err_clk;
	}
	pce_dev->ce_clk = ce_clk;

	/* Enable CE core clk */
	rc = clk_prepare_enable(pce_dev->ce_core_clk);
	if (rc) {
		pr_err("Unable to enable/prepare CE core clk\n");
		if (pce_dev->ce_core_src_clk != NULL)
			clk_put(pce_dev->ce_core_src_clk);
		clk_put(pce_dev->ce_core_clk);
		clk_put(pce_dev->ce_clk);
		goto err_clk;
	} else {
		/* Enable CE clk */
		rc = clk_prepare_enable(pce_dev->ce_clk);
		if (rc) {
			pr_err("Unable to enable/prepare CE iface clk\n");
			clk_disable_unprepare(pce_dev->ce_core_clk);
			if (pce_dev->ce_core_src_clk != NULL)
				clk_put(pce_dev->ce_core_src_clk);
			clk_put(pce_dev->ce_core_clk);
			clk_put(pce_dev->ce_clk);
			goto err_clk;
		}
	}
err_clk:
	if (rc)
		pr_err("Unable to init CE clks, rc = %d\n", rc);
	return rc;
}

/* crypto engine open function. */
void *qce_open(struct platform_device *pdev, int *rc)
{
	struct qce_device *pce_dev;

	pce_dev = kzalloc(sizeof(struct qce_device), GFP_KERNEL);
	if (!pce_dev) {
		*rc = -ENOMEM;
		pr_err("Can not allocate memory: %d\n", *rc);
		return NULL;
	}
	pce_dev->pdev = &pdev->dev;

	if (pdev->dev.of_node) {
		*rc = __qce_get_device_tree_data(pdev, pce_dev);
		if (*rc)
			goto err_pce_dev;
	} else {
		*rc = -EINVAL;
		pr_err("Device Node not found.\n");
		goto err_pce_dev;
	}

	pce_dev->memsize = 9 * PAGE_SIZE;
	pce_dev->coh_vmem = dma_alloc_coherent(pce_dev->pdev,
			pce_dev->memsize, &pce_dev->coh_pmem, GFP_KERNEL);
	if (pce_dev->coh_vmem == NULL) {
		*rc = -ENOMEM;
		pr_err("Can not allocate coherent memory for sps data\n");
		goto err_iobase;
	}

	*rc = __qce_init_clk(pce_dev);
	if (*rc)
		goto err_mem;

	if (_probe_ce_engine(pce_dev)) {
		*rc = -ENXIO;
		goto err;
	}
	*rc = 0;
	qce_setup_ce_sps_data(pce_dev);
	qce_sps_init(pce_dev);

	return pce_dev;
err:
	clk_disable_unprepare(pce_dev->ce_clk);
	clk_disable_unprepare(pce_dev->ce_core_clk);

	if (pce_dev->ce_core_src_clk != NULL)
		clk_put(pce_dev->ce_core_src_clk);
	clk_put(pce_dev->ce_clk);
	clk_put(pce_dev->ce_core_clk);
err_mem:
	if (pce_dev->coh_vmem)
		dma_free_coherent(pce_dev->pdev, pce_dev->memsize,
			pce_dev->coh_vmem, pce_dev->coh_pmem);
err_iobase:
	if (pce_dev->ce_sps.bam_iobase)
		iounmap(pce_dev->ce_sps.bam_iobase);
	if (pce_dev->iobase)
		iounmap(pce_dev->iobase);
err_pce_dev:
	kfree(pce_dev);
	return NULL;
}
EXPORT_SYMBOL(qce_open);

/* crypto engine close function. */
int qce_close(void *handle)
{
	struct qce_device *pce_dev = (struct qce_device *) handle;

	if (handle == NULL)
		return -ENODEV;

	if (pce_dev->iobase)
		iounmap(pce_dev->iobase);
	if (pce_dev->coh_vmem)
		dma_free_coherent(pce_dev->pdev, pce_dev->memsize,
				pce_dev->coh_vmem, pce_dev->coh_pmem);

	clk_disable_unprepare(pce_dev->ce_clk);
	clk_disable_unprepare(pce_dev->ce_core_clk);
	if (pce_dev->ce_core_src_clk != NULL)
		clk_put(pce_dev->ce_core_src_clk);
	clk_put(pce_dev->ce_clk);
	clk_put(pce_dev->ce_core_clk);

	qce_sps_exit(pce_dev);
	kfree(handle);

	return 0;
}
EXPORT_SYMBOL(qce_close);

int qce_hw_support(void *handle, struct ce_hw_support *ce_support)
{
	struct qce_device *pce_dev = (struct qce_device *)handle;

	if (ce_support == NULL)
		return -EINVAL;

	ce_support->sha1_hmac_20 = false;
	ce_support->sha1_hmac = false;
	ce_support->sha256_hmac = false;
	ce_support->sha_hmac = true;
	ce_support->cmac  = true;
	ce_support->aes_key_192 = false;
	ce_support->aes_xts = true;
	ce_support->ota = false;
	ce_support->bam = true;
	if (pce_dev->ce_sps.minor_version) {
		ce_support->aligned_only = false;
		ce_support->aes_ccm = true;
	} else {
		ce_support->aligned_only = true;
		ce_support->aes_ccm = false;
	}
	return 0;
}
EXPORT_SYMBOL(qce_hw_support);

static int __init qce_init(void)
{
	bam_registry.handle = 0;
	bam_registry.cnt = 0;
	return 0;
}

static void __exit qce_exit(void)
{
	bam_registry.handle = 0;
	bam_registry.cnt = 0;
}

module_init(qce_init);
module_exit(qce_exit);

MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Crypto Engine driver");