// SPDX-License-Identifier: GPL-2.0-or-later /* * Driver for IBM Power 842 compression accelerator * * Copyright (C) IBM Corporation, 2012 * * Authors: Robert Jennings * Seth Jennings */ #include #include #include #include "nx-842.h" #include "nx_csbcpb.h" /* struct nx_csbcpb */ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Robert Jennings "); MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); MODULE_ALIAS_CRYPTO("842"); MODULE_ALIAS_CRYPTO("842-nx"); /* * Coprocessor type specific capabilities from the hypervisor. */ struct hv_nx_cop_caps { __be64 descriptor; __be64 req_max_processed_len; /* Max bytes in one GZIP request */ __be64 min_compress_len; /* Min compression size in bytes */ __be64 min_decompress_len; /* Min decompression size in bytes */ } __packed __aligned(0x1000); /* * Coprocessor type specific capabilities. */ struct nx_cop_caps { u64 descriptor; u64 req_max_processed_len; /* Max bytes in one GZIP request */ u64 min_compress_len; /* Min compression in bytes */ u64 min_decompress_len; /* Min decompression in bytes */ }; static u64 caps_feat; static struct nx_cop_caps nx_cop_caps; static struct nx842_constraints nx842_pseries_constraints = { .alignment = DDE_BUFFER_ALIGN, .multiple = DDE_BUFFER_LAST_MULT, .minimum = DDE_BUFFER_LAST_MULT, .maximum = PAGE_SIZE, /* dynamic, max_sync_size */ }; static int check_constraints(unsigned long buf, unsigned int *len, bool in) { if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) { pr_debug("%s buffer 0x%lx not aligned to 0x%x\n", in ? "input" : "output", buf, nx842_pseries_constraints.alignment); return -EINVAL; } if (*len % nx842_pseries_constraints.multiple) { pr_debug("%s buffer len 0x%x not multiple of 0x%x\n", in ? "input" : "output", *len, nx842_pseries_constraints.multiple); if (in) return -EINVAL; *len = round_down(*len, nx842_pseries_constraints.multiple); } if (*len < nx842_pseries_constraints.minimum) { pr_debug("%s buffer len 0x%x under minimum 0x%x\n", in ? "input" : "output", *len, nx842_pseries_constraints.minimum); return -EINVAL; } if (*len > nx842_pseries_constraints.maximum) { pr_debug("%s buffer len 0x%x over maximum 0x%x\n", in ? "input" : "output", *len, nx842_pseries_constraints.maximum); if (in) return -EINVAL; *len = nx842_pseries_constraints.maximum; } return 0; } /* I assume we need to align the CSB? */ #define WORKMEM_ALIGN (256) struct nx842_workmem { /* scatterlist */ char slin[4096]; char slout[4096]; /* coprocessor status/parameter block */ struct nx_csbcpb csbcpb; char padding[WORKMEM_ALIGN]; } __aligned(WORKMEM_ALIGN); /* Macros for fields within nx_csbcpb */ /* Check the valid bit within the csbcpb valid field */ #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7)) /* CE macros operate on the completion_extension field bits in the csbcpb. * CE0 0=full completion, 1=partial completion * CE1 0=CE0 indicates completion, 1=termination (output may be modified) * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */ #define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7)) #define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6)) #define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5)) /* The NX unit accepts data only on 4K page boundaries */ #define NX842_HW_PAGE_SIZE (4096) #define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1)) struct ibm_nx842_counters { atomic64_t comp_complete; atomic64_t comp_failed; atomic64_t decomp_complete; atomic64_t decomp_failed; atomic64_t swdecomp; atomic64_t comp_times[32]; atomic64_t decomp_times[32]; }; struct nx842_devdata { struct vio_dev *vdev; struct device *dev; struct ibm_nx842_counters *counters; unsigned int max_sg_len; unsigned int max_sync_size; unsigned int max_sync_sg; }; static struct nx842_devdata __rcu *devdata; static DEFINE_SPINLOCK(devdata_spinlock); #define NX842_COUNTER_INC(_x) \ static inline void nx842_inc_##_x( \ const struct nx842_devdata *dev) { \ if (dev) \ atomic64_inc(&dev->counters->_x); \ } NX842_COUNTER_INC(comp_complete); NX842_COUNTER_INC(comp_failed); NX842_COUNTER_INC(decomp_complete); NX842_COUNTER_INC(decomp_failed); NX842_COUNTER_INC(swdecomp); #define NX842_HIST_SLOTS 16 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time) { int bucket = fls(time); if (bucket) bucket = min((NX842_HIST_SLOTS - 1), bucket - 1); atomic64_inc(×[bucket]); } /* NX unit operation flags */ #define NX842_OP_COMPRESS 0x0 #define NX842_OP_CRC 0x1 #define NX842_OP_DECOMPRESS 0x2 #define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC) #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC) #define NX842_OP_ASYNC (1<<23) #define NX842_OP_NOTIFY (1<<22) #define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8) static unsigned long nx842_get_desired_dma(struct vio_dev *viodev) { /* No use of DMA mappings within the driver. */ return 0; } struct nx842_slentry { __be64 ptr; /* Real address (use __pa()) */ __be64 len; }; /* pHyp scatterlist entry */ struct nx842_scatterlist { int entry_nr; /* number of slentries */ struct nx842_slentry *entries; /* ptr to array of slentries */ }; /* Does not include sizeof(entry_nr) in the size */ static inline unsigned long nx842_get_scatterlist_size( struct nx842_scatterlist *sl) { return sl->entry_nr * sizeof(struct nx842_slentry); } static int nx842_build_scatterlist(unsigned long buf, int len, struct nx842_scatterlist *sl) { unsigned long entrylen; struct nx842_slentry *entry; sl->entry_nr = 0; entry = sl->entries; while (len) { entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf)); entrylen = min_t(int, len, LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE)); entry->len = cpu_to_be64(entrylen); len -= entrylen; buf += entrylen; sl->entry_nr++; entry++; } return 0; } static int nx842_validate_result(struct device *dev, struct cop_status_block *csb) { /* The csb must be valid after returning from vio_h_cop_sync */ if (!NX842_CSBCBP_VALID_CHK(csb->valid)) { dev_err(dev, "%s: cspcbp not valid upon completion.\n", __func__); dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n", csb->valid, csb->crb_seq_number, csb->completion_code, csb->completion_extension); dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n", be32_to_cpu(csb->processed_byte_count), (unsigned long)be64_to_cpu(csb->address)); return -EIO; } /* Check return values from the hardware in the CSB */ switch (csb->completion_code) { case 0: /* Completed without error */ break; case 64: /* Compression ok, but output larger than input */ dev_dbg(dev, "%s: output size larger than input size\n", __func__); break; case 13: /* Output buffer too small */ dev_dbg(dev, "%s: Out of space in output buffer\n", __func__); return -ENOSPC; case 65: /* Calculated CRC doesn't match the passed value */ dev_dbg(dev, "%s: CRC mismatch for decompression\n", __func__); return -EINVAL; case 66: /* Input data contains an illegal template field */ case 67: /* Template indicates data past the end of the input stream */ dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n", __func__, csb->completion_code); return -EINVAL; default: dev_dbg(dev, "%s: Unspecified error (code:%d)\n", __func__, csb->completion_code); return -EIO; } /* Hardware sanity check */ if (!NX842_CSBCPB_CE2(csb->completion_extension)) { dev_err(dev, "%s: No error returned by hardware, but " "data returned is unusable, contact support.\n" "(Additional info: csbcbp->processed bytes " "does not specify processed bytes for the " "target buffer.)\n", __func__); return -EIO; } return 0; } /** * nx842_pseries_compress - Compress data using the 842 algorithm * * Compression provide by the NX842 coprocessor on IBM Power systems. * The input buffer is compressed and the result is stored in the * provided output buffer. * * Upon return from this function @outlen contains the length of the * compressed data. If there is an error then @outlen will be 0 and an * error will be specified by the return code from this function. * * @in: Pointer to input buffer * @inlen: Length of input buffer * @out: Pointer to output buffer * @outlen: Length of output buffer * @wmem: ptr to buffer for working memory, size determined by * nx842_pseries_driver.workmem_size * * Returns: * 0 Success, output of length @outlen stored in the buffer at @out * -ENOMEM Unable to allocate internal buffers * -ENOSPC Output buffer is to small * -EIO Internal error * -ENODEV Hardware unavailable */ static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlen, void *wmem) { struct nx842_devdata *local_devdata; struct device *dev = NULL; struct nx842_workmem *workmem; struct nx842_scatterlist slin, slout; struct nx_csbcpb *csbcpb; int ret = 0; unsigned long inbuf, outbuf; struct vio_pfo_op op = { .done = NULL, .handle = 0, .timeout = 0, }; unsigned long start = get_tb(); inbuf = (unsigned long)in; if (check_constraints(inbuf, &inlen, true)) return -EINVAL; outbuf = (unsigned long)out; if (check_constraints(outbuf, outlen, false)) return -EINVAL; rcu_read_lock(); local_devdata = rcu_dereference(devdata); if (!local_devdata || !local_devdata->dev) { rcu_read_unlock(); return -ENODEV; } dev = local_devdata->dev; /* Init scatterlist */ workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); slin.entries = (struct nx842_slentry *)workmem->slin; slout.entries = (struct nx842_slentry *)workmem->slout; /* Init operation */ op.flags = NX842_OP_COMPRESS_CRC; csbcpb = &workmem->csbcpb; memset(csbcpb, 0, sizeof(*csbcpb)); op.csbcpb = nx842_get_pa(csbcpb); if ((inbuf & NX842_HW_PAGE_MASK) == ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { /* Create direct DDE */ op.in = nx842_get_pa((void *)inbuf); op.inlen = inlen; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(inbuf, inlen, &slin); op.in = nx842_get_pa(slin.entries); op.inlen = -nx842_get_scatterlist_size(&slin); } if ((outbuf & NX842_HW_PAGE_MASK) == ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { /* Create direct DDE */ op.out = nx842_get_pa((void *)outbuf); op.outlen = *outlen; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(outbuf, *outlen, &slout); op.out = nx842_get_pa(slout.entries); op.outlen = -nx842_get_scatterlist_size(&slout); } dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", __func__, (unsigned long)op.in, (long)op.inlen, (unsigned long)op.out, (long)op.outlen); /* Send request to pHyp */ ret = vio_h_cop_sync(local_devdata->vdev, &op); /* Check for pHyp error */ if (ret) { dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", __func__, ret, op.hcall_err); ret = -EIO; goto unlock; } /* Check for hardware error */ ret = nx842_validate_result(dev, &csbcpb->csb); if (ret) goto unlock; *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen); unlock: if (ret) nx842_inc_comp_failed(local_devdata); else { nx842_inc_comp_complete(local_devdata); ibm_nx842_incr_hist(local_devdata->counters->comp_times, (get_tb() - start) / tb_ticks_per_usec); } rcu_read_unlock(); return ret; } /** * nx842_pseries_decompress - Decompress data using the 842 algorithm * * Decompression provide by the NX842 coprocessor on IBM Power systems. * The input buffer is decompressed and the result is stored in the * provided output buffer. The size allocated to the output buffer is * provided by the caller of this function in @outlen. Upon return from * this function @outlen contains the length of the decompressed data. * If there is an error then @outlen will be 0 and an error will be * specified by the return code from this function. * * @in: Pointer to input buffer * @inlen: Length of input buffer * @out: Pointer to output buffer * @outlen: Length of output buffer * @wmem: ptr to buffer for working memory, size determined by * nx842_pseries_driver.workmem_size * * Returns: * 0 Success, output of length @outlen stored in the buffer at @out * -ENODEV Hardware decompression device is unavailable * -ENOMEM Unable to allocate internal buffers * -ENOSPC Output buffer is to small * -EINVAL Bad input data encountered when attempting decompress * -EIO Internal error */ static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlen, void *wmem) { struct nx842_devdata *local_devdata; struct device *dev = NULL; struct nx842_workmem *workmem; struct nx842_scatterlist slin, slout; struct nx_csbcpb *csbcpb; int ret = 0; unsigned long inbuf, outbuf; struct vio_pfo_op op = { .done = NULL, .handle = 0, .timeout = 0, }; unsigned long start = get_tb(); /* Ensure page alignment and size */ inbuf = (unsigned long)in; if (check_constraints(inbuf, &inlen, true)) return -EINVAL; outbuf = (unsigned long)out; if (check_constraints(outbuf, outlen, false)) return -EINVAL; rcu_read_lock(); local_devdata = rcu_dereference(devdata); if (!local_devdata || !local_devdata->dev) { rcu_read_unlock(); return -ENODEV; } dev = local_devdata->dev; workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); /* Init scatterlist */ slin.entries = (struct nx842_slentry *)workmem->slin; slout.entries = (struct nx842_slentry *)workmem->slout; /* Init operation */ op.flags = NX842_OP_DECOMPRESS_CRC; csbcpb = &workmem->csbcpb; memset(csbcpb, 0, sizeof(*csbcpb)); op.csbcpb = nx842_get_pa(csbcpb); if ((inbuf & NX842_HW_PAGE_MASK) == ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { /* Create direct DDE */ op.in = nx842_get_pa((void *)inbuf); op.inlen = inlen; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(inbuf, inlen, &slin); op.in = nx842_get_pa(slin.entries); op.inlen = -nx842_get_scatterlist_size(&slin); } if ((outbuf & NX842_HW_PAGE_MASK) == ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { /* Create direct DDE */ op.out = nx842_get_pa((void *)outbuf); op.outlen = *outlen; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(outbuf, *outlen, &slout); op.out = nx842_get_pa(slout.entries); op.outlen = -nx842_get_scatterlist_size(&slout); } dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", __func__, (unsigned long)op.in, (long)op.inlen, (unsigned long)op.out, (long)op.outlen); /* Send request to pHyp */ ret = vio_h_cop_sync(local_devdata->vdev, &op); /* Check for pHyp error */ if (ret) { dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", __func__, ret, op.hcall_err); goto unlock; } /* Check for hardware error */ ret = nx842_validate_result(dev, &csbcpb->csb); if (ret) goto unlock; *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); unlock: if (ret) /* decompress fail */ nx842_inc_decomp_failed(local_devdata); else { nx842_inc_decomp_complete(local_devdata); ibm_nx842_incr_hist(local_devdata->counters->decomp_times, (get_tb() - start) / tb_ticks_per_usec); } rcu_read_unlock(); return ret; } /** * nx842_OF_set_defaults -- Set default (disabled) values for devdata * * @devdata: struct nx842_devdata to update * * Returns: * 0 on success * -ENOENT if @devdata ptr is NULL */ static int nx842_OF_set_defaults(struct nx842_devdata *devdata) { if (devdata) { devdata->max_sync_size = 0; devdata->max_sync_sg = 0; devdata->max_sg_len = 0; return 0; } else return -ENOENT; } /** * nx842_OF_upd_status -- Check the device info from OF status prop * * The status property indicates if the accelerator is enabled. If the * device is in the OF tree it indicates that the hardware is present. * The status field indicates if the device is enabled when the status * is 'okay'. Otherwise the device driver will be disabled. * * @devdata: struct nx842_devdata to use for dev_info * @prop: struct property point containing the maxsyncop for the update * * Returns: * 0 - Device is available * -ENODEV - Device is not available */ static int nx842_OF_upd_status(struct nx842_devdata *devdata, struct property *prop) { const char *status = (const char *)prop->value; if (!strncmp(status, "okay", (size_t)prop->length)) return 0; if (!strncmp(status, "disabled", (size_t)prop->length)) return -ENODEV; dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status); return -EINVAL; } /** * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop * * Definition of the 'ibm,max-sg-len' OF property: * This field indicates the maximum byte length of a scatter list * for the platform facility. It is a single cell encoded as with encode-int. * * Example: * # od -x ibm,max-sg-len * 0000000 0000 0ff0 * * In this example, the maximum byte length of a scatter list is * 0x0ff0 (4,080). * * @devdata: struct nx842_devdata to update * @prop: struct property point containing the maxsyncop for the update * * Returns: * 0 on success * -EINVAL on failure */ static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata, struct property *prop) { int ret = 0; const unsigned int maxsglen = of_read_number(prop->value, 1); if (prop->length != sizeof(maxsglen)) { dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__); dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__, prop->length, sizeof(maxsglen)); ret = -EINVAL; } else { devdata->max_sg_len = min_t(unsigned int, maxsglen, NX842_HW_PAGE_SIZE); } return ret; } /** * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop * * Definition of the 'ibm,max-sync-cop' OF property: * Two series of cells. The first series of cells represents the maximums * that can be synchronously compressed. The second series of cells * represents the maximums that can be synchronously decompressed. * 1. The first cell in each series contains the count of the number of * data length, scatter list elements pairs that follow – each being * of the form * a. One cell data byte length * b. One cell total number of scatter list elements * * Example: * # od -x ibm,max-sync-cop * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001 * 0000020 0000 1000 0000 01fe * * In this example, compression supports 0x1000 (4,096) data byte length * and 0x1fe (510) total scatter list elements. Decompression supports * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list * elements. * * @devdata: struct nx842_devdata to update * @prop: struct property point containing the maxsyncop for the update * * Returns: * 0 on success * -EINVAL on failure */ static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata, struct property *prop) { int ret = 0; unsigned int comp_data_limit, decomp_data_limit; unsigned int comp_sg_limit, decomp_sg_limit; const struct maxsynccop_t { __be32 comp_elements; __be32 comp_data_limit; __be32 comp_sg_limit; __be32 decomp_elements; __be32 decomp_data_limit; __be32 decomp_sg_limit; } *maxsynccop; if (prop->length != sizeof(*maxsynccop)) { dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__); dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length, sizeof(*maxsynccop)); ret = -EINVAL; goto out; } maxsynccop = (const struct maxsynccop_t *)prop->value; comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit); comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit); decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit); decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit); /* Use one limit rather than separate limits for compression and * decompression. Set a maximum for this so as not to exceed the * size that the header can support and round the value down to * the hardware page size (4K) */ devdata->max_sync_size = min(comp_data_limit, decomp_data_limit); devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size, 65536); if (devdata->max_sync_size < 4096) { dev_err(devdata->dev, "%s: hardware max data size (%u) is " "less than the driver minimum, unable to use " "the hardware device\n", __func__, devdata->max_sync_size); ret = -EINVAL; goto out; } nx842_pseries_constraints.maximum = devdata->max_sync_size; devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit); if (devdata->max_sync_sg < 1) { dev_err(devdata->dev, "%s: hardware max sg size (%u) is " "less than the driver minimum, unable to use " "the hardware device\n", __func__, devdata->max_sync_sg); ret = -EINVAL; goto out; } out: return ret; } /** * nx842_OF_upd -- Handle OF properties updates for the device. * * Set all properties from the OF tree. Optionally, a new property * can be provided by the @new_prop pointer to overwrite an existing value. * The device will remain disabled until all values are valid, this function * will return an error for updates unless all values are valid. * * @new_prop: If not NULL, this property is being updated. If NULL, update * all properties from the current values in the OF tree. * * Returns: * 0 - Success * -ENOMEM - Could not allocate memory for new devdata structure * -EINVAL - property value not found, new_prop is not a recognized * property for the device or property value is not valid. * -ENODEV - Device is not available */ static int nx842_OF_upd(struct property *new_prop) { struct nx842_devdata *old_devdata = NULL; struct nx842_devdata *new_devdata = NULL; struct device_node *of_node = NULL; struct property *status = NULL; struct property *maxsglen = NULL; struct property *maxsyncop = NULL; int ret = 0; unsigned long flags; new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); if (!new_devdata) return -ENOMEM; spin_lock_irqsave(&devdata_spinlock, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_spinlock)); if (old_devdata) of_node = old_devdata->dev->of_node; if (!old_devdata || !of_node) { pr_err("%s: device is not available\n", __func__); spin_unlock_irqrestore(&devdata_spinlock, flags); kfree(new_devdata); return -ENODEV; } memcpy(new_devdata, old_devdata, sizeof(*old_devdata)); new_devdata->counters = old_devdata->counters; /* Set ptrs for existing properties */ status = of_find_property(of_node, "status", NULL); maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL); maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL); if (!status || !maxsglen || !maxsyncop) { dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__); ret = -EINVAL; goto error_out; } /* * If this is a property update, there are only certain properties that * we care about. Bail if it isn't in the below list */ if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) || strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) || strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length))) goto out; /* Perform property updates */ ret = nx842_OF_upd_status(new_devdata, status); if (ret) goto error_out; ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen); if (ret) goto error_out; ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop); if (ret) goto error_out; out: dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n", __func__, new_devdata->max_sync_size, old_devdata->max_sync_size); dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n", __func__, new_devdata->max_sync_sg, old_devdata->max_sync_sg); dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n", __func__, new_devdata->max_sg_len, old_devdata->max_sg_len); rcu_assign_pointer(devdata, new_devdata); spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); dev_set_drvdata(new_devdata->dev, new_devdata); kfree(old_devdata); return 0; error_out: if (new_devdata) { dev_info(old_devdata->dev, "%s: device disabled\n", __func__); nx842_OF_set_defaults(new_devdata); rcu_assign_pointer(devdata, new_devdata); spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); dev_set_drvdata(new_devdata->dev, new_devdata); kfree(old_devdata); } else { dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__); spin_unlock_irqrestore(&devdata_spinlock, flags); } if (!ret) ret = -EINVAL; return ret; } /** * nx842_OF_notifier - Process updates to OF properties for the device * * @np: notifier block * @action: notifier action * @data: struct of_reconfig_data pointer * * Returns: * NOTIFY_OK on success * NOTIFY_BAD encoded with error number on failure, use * notifier_to_errno() to decode this value */ static int nx842_OF_notifier(struct notifier_block *np, unsigned long action, void *data) { struct of_reconfig_data *upd = data; struct nx842_devdata *local_devdata; struct device_node *node = NULL; rcu_read_lock(); local_devdata = rcu_dereference(devdata); if (local_devdata) node = local_devdata->dev->of_node; if (local_devdata && action == OF_RECONFIG_UPDATE_PROPERTY && !strcmp(upd->dn->name, node->name)) { rcu_read_unlock(); nx842_OF_upd(upd->prop); } else rcu_read_unlock(); return NOTIFY_OK; } static struct notifier_block nx842_of_nb = { .notifier_call = nx842_OF_notifier, }; #define nx842_counter_read(_name) \ static ssize_t nx842_##_name##_show(struct device *dev, \ struct device_attribute *attr, \ char *buf) { \ struct nx842_devdata *local_devdata; \ int p = 0; \ rcu_read_lock(); \ local_devdata = rcu_dereference(devdata); \ if (local_devdata) \ p = snprintf(buf, PAGE_SIZE, "%lld\n", \ atomic64_read(&local_devdata->counters->_name)); \ rcu_read_unlock(); \ return p; \ } #define NX842DEV_COUNTER_ATTR_RO(_name) \ nx842_counter_read(_name); \ static struct device_attribute dev_attr_##_name = __ATTR(_name, \ 0444, \ nx842_##_name##_show,\ NULL); NX842DEV_COUNTER_ATTR_RO(comp_complete); NX842DEV_COUNTER_ATTR_RO(comp_failed); NX842DEV_COUNTER_ATTR_RO(decomp_complete); NX842DEV_COUNTER_ATTR_RO(decomp_failed); NX842DEV_COUNTER_ATTR_RO(swdecomp); static ssize_t nx842_timehist_show(struct device *, struct device_attribute *, char *); static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444, nx842_timehist_show, NULL); static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times, 0444, nx842_timehist_show, NULL); static ssize_t nx842_timehist_show(struct device *dev, struct device_attribute *attr, char *buf) { char *p = buf; struct nx842_devdata *local_devdata; atomic64_t *times; int bytes_remain = PAGE_SIZE; int bytes; int i; rcu_read_lock(); local_devdata = rcu_dereference(devdata); if (!local_devdata) { rcu_read_unlock(); return 0; } if (attr == &dev_attr_comp_times) times = local_devdata->counters->comp_times; else if (attr == &dev_attr_decomp_times) times = local_devdata->counters->decomp_times; else { rcu_read_unlock(); return 0; } for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) { bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n", i ? (2<<(i-1)) : 0, (2<counters = kzalloc(sizeof(*new_devdata->counters), GFP_NOFS); if (!new_devdata->counters) { kfree(new_devdata); return -ENOMEM; } spin_lock_irqsave(&devdata_spinlock, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_spinlock)); if (old_devdata && old_devdata->vdev != NULL) { dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__); ret = -1; goto error_unlock; } dev_set_drvdata(&viodev->dev, NULL); new_devdata->vdev = viodev; new_devdata->dev = &viodev->dev; nx842_OF_set_defaults(new_devdata); rcu_assign_pointer(devdata, new_devdata); spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); kfree(old_devdata); of_reconfig_notifier_register(&nx842_of_nb); ret = nx842_OF_upd(NULL); if (ret) goto error; ret = crypto_register_alg(&nx842_pseries_alg); if (ret) { dev_err(&viodev->dev, "could not register comp alg: %d\n", ret); goto error; } rcu_read_lock(); dev_set_drvdata(&viodev->dev, rcu_dereference(devdata)); rcu_read_unlock(); if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) { dev_err(&viodev->dev, "could not create sysfs device attributes\n"); ret = -1; goto error; } if (caps_feat) { if (sysfs_create_group(&viodev->dev.kobj, &nxcop_caps_attr_group)) { dev_err(&viodev->dev, "Could not create sysfs NX capability entries\n"); ret = -1; goto error; } } return 0; error_unlock: spin_unlock_irqrestore(&devdata_spinlock, flags); if (new_devdata) kfree(new_devdata->counters); kfree(new_devdata); error: return ret; } static void nx842_remove(struct vio_dev *viodev) { struct nx842_devdata *old_devdata; unsigned long flags; pr_info("Removing IBM Power 842 compression device\n"); sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group); if (caps_feat) sysfs_remove_group(&viodev->dev.kobj, &nxcop_caps_attr_group); crypto_unregister_alg(&nx842_pseries_alg); of_reconfig_notifier_unregister(&nx842_of_nb); spin_lock_irqsave(&devdata_spinlock, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_spinlock)); RCU_INIT_POINTER(devdata, NULL); spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); dev_set_drvdata(&viodev->dev, NULL); if (old_devdata) kfree(old_devdata->counters); kfree(old_devdata); } /* * Get NX capabilities from the hypervisor. * Only NXGZIP capabilities are provided by the hypersvisor right * now and these values are available to user space with sysfs. */ static void __init nxcop_get_capabilities(void) { struct hv_vas_all_caps *hv_caps; struct hv_nx_cop_caps *hv_nxc; int rc; hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL); if (!hv_caps) return; /* * Get NX overall capabilities with feature type=0 */ rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0, (u64)virt_to_phys(hv_caps)); if (rc) goto out; caps_feat = be64_to_cpu(hv_caps->feat_type); /* * NX-GZIP feature available */ if (caps_feat & VAS_NX_GZIP_FEAT_BIT) { hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL); if (!hv_nxc) goto out; /* * Get capabilities for NX-GZIP feature */ rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, VAS_NX_GZIP_FEAT, (u64)virt_to_phys(hv_nxc)); } else { pr_err("NX-GZIP feature is not available\n"); rc = -EINVAL; } if (!rc) { nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor); nx_cop_caps.req_max_processed_len = be64_to_cpu(hv_nxc->req_max_processed_len); nx_cop_caps.min_compress_len = be64_to_cpu(hv_nxc->min_compress_len); nx_cop_caps.min_decompress_len = be64_to_cpu(hv_nxc->min_decompress_len); } else { caps_feat = 0; } kfree(hv_nxc); out: kfree(hv_caps); } static const struct vio_device_id nx842_vio_driver_ids[] = { {"ibm,compression-v1", "ibm,compression"}, {"", ""}, }; MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids); static struct vio_driver nx842_vio_driver = { .name = KBUILD_MODNAME, .probe = nx842_probe, .remove = nx842_remove, .get_desired_dma = nx842_get_desired_dma, .id_table = nx842_vio_driver_ids, }; static int __init nx842_pseries_init(void) { struct nx842_devdata *new_devdata; struct device_node *np; int ret; np = of_find_compatible_node(NULL, NULL, "ibm,compression"); if (!np) return -ENODEV; of_node_put(np); RCU_INIT_POINTER(devdata, NULL); new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL); if (!new_devdata) return -ENOMEM; RCU_INIT_POINTER(devdata, new_devdata); /* * Get NX capabilities from the hypervisor. */ nxcop_get_capabilities(); ret = vio_register_driver(&nx842_vio_driver); if (ret) { pr_err("Could not register VIO driver %d\n", ret); kfree(new_devdata); return ret; } ret = vas_register_api_pseries(THIS_MODULE, VAS_COP_TYPE_GZIP, "nx-gzip"); if (ret) pr_err("NX-GZIP is not supported. Returned=%d\n", ret); return 0; } module_init(nx842_pseries_init); static void __exit nx842_pseries_exit(void) { struct nx842_devdata *old_devdata; unsigned long flags; vas_unregister_api_pseries(); crypto_unregister_alg(&nx842_pseries_alg); spin_lock_irqsave(&devdata_spinlock, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_spinlock)); RCU_INIT_POINTER(devdata, NULL); spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); if (old_devdata && old_devdata->dev) dev_set_drvdata(old_devdata->dev, NULL); kfree(old_devdata); vio_unregister_driver(&nx842_vio_driver); } module_exit(nx842_pseries_exit);