Index: head/share/man/man4/mpr.4 =================================================================== --- head/share/man/man4/mpr.4 +++ head/share/man/man4/mpr.4 @@ -1,8 +1,8 @@ .\" .\" Copyright (c) 2010 Spectra Logic Corporation .\" Copyright (c) 2014 LSI Corp -.\" Copyright (c) 2017 Avago Technologies -.\" Copyright (c) 2017 Broadcom Ltd. +.\" Copyright (c) 2015-2017 Avago Technologies +.\" Copyright (c) 2015-2017 Broadcom Ltd. .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -38,7 +38,7 @@ .\" $Id$ .\" $FreeBSD$ .\" -.Dd May 17, 2017 +.Dd May 25, 2017 .Dt MPR 4 .Os .Sh NAME Index: head/sys/dev/mpr/mpr.c =================================================================== --- head/sys/dev/mpr/mpr.c +++ head/sys/dev/mpr/mpr.c @@ -520,7 +520,8 @@ */ if (reallocating) { mpr_iocfacts_free(sc); - mprsas_realloc_targets(sc, saved_facts.MaxTargets); + mprsas_realloc_targets(sc, saved_facts.MaxTargets + + saved_facts.MaxVolumes); } /* @@ -1663,6 +1664,7 @@ mtx_init(&sc->mpr_mtx, "MPR lock", NULL, MTX_DEF); callout_init_mtx(&sc->periodic, &sc->mpr_mtx, 0); + callout_init_mtx(&sc->device_check_callout, &sc->mpr_mtx, 0); TAILQ_INIT(&sc->event_list); timevalclear(&sc->lastfail); @@ -1832,6 +1834,7 @@ mpr_unlock(sc); /* Lock must not be held for this */ callout_drain(&sc->periodic); + callout_drain(&sc->device_check_callout); if (((error = mpr_detach_log(sc)) != 0) || ((error = mpr_detach_sas(sc)) != 0)) Index: head/sys/dev/mpr/mpr_mapping.c =================================================================== --- head/sys/dev/mpr/mpr_mapping.c +++ head/sys/dev/mpr/mpr_mapping.c @@ -60,7 +60,7 @@ #include /** - * _mapping_clear_entry - Clear a particular mapping entry. + * _mapping_clear_map_entry - Clear a particular mapping entry. * @map_entry: map table entry * * Returns nothing. @@ -73,7 +73,6 @@ map_entry->phy_bits = 0; map_entry->dpm_entry_num = MPR_DPM_BAD_IDX; map_entry->dev_handle = 0; - map_entry->channel = -1; map_entry->id = -1; map_entry->missing_count = 0; map_entry->init_complete = 0; @@ -140,12 +139,15 @@ dpm_entry->PhysicalBitsMapping = htole32(et_entry->phy_bits); dpm_entry->Reserved1 = 0; + mpr_dprint(sc, MPR_MAPPING, "%s: Writing DPM entry %d for enclosure.\n", + __func__, et_entry->dpm_entry_num); memcpy(&config_page.Entry, (u8 *)dpm_entry, sizeof(Mpi2DriverMap0Entry_t)); if (mpr_config_set_dpm_pg0(sc, &mpi_reply, &config_page, et_entry->dpm_entry_num)) { - printf("%s: write of dpm entry %d for enclosure failed\n", - __func__, et_entry->dpm_entry_num); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: Write of DPM " + "entry %d for enclosure failed.\n", __func__, + et_entry->dpm_entry_num); dpm_entry->MappingInformation = le16toh(dpm_entry-> MappingInformation); dpm_entry->DeviceIndex = le16toh(dpm_entry->DeviceIndex); @@ -164,7 +166,7 @@ /** * _mapping_commit_map_entry - write a particular map table entry in DPM page0. * @sc: per adapter object - * @enc_entry: enclosure table entry + * @mt_entry: mapping table entry * * Returns 0 for success, non-zero for failure. */ @@ -180,6 +182,19 @@ if (!sc->is_dpm_enable) return 0; + /* + * It's possible that this Map Entry points to a BAD DPM index. This + * can happen if the Map Entry is a for a missing device and the DPM + * entry that was being used by this device is now being used by some + * new device. So, check for a BAD DPM index and just return if so. + */ + if (mt_entry->dpm_entry_num == MPR_DPM_BAD_IDX) { + mpr_dprint(sc, MPR_MAPPING, "%s: DPM entry location for target " + "%d is invalid. DPM will not be written.\n", __func__, + mt_entry->id); + return 0; + } + memset(&config_page, 0, sizeof(Mpi2DriverMappingPage0_t)); memcpy(&config_page.Header, (u8 *)sc->dpm_pg0, sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); @@ -193,13 +208,16 @@ dpm_entry->MappingInformation = htole16(mt_entry->missing_count); dpm_entry->PhysicalBitsMapping = 0; dpm_entry->Reserved1 = 0; - dpm_entry->MappingInformation = htole16(dpm_entry->MappingInformation); memcpy(&config_page.Entry, (u8 *)dpm_entry, sizeof(Mpi2DriverMap0Entry_t)); + + mpr_dprint(sc, MPR_MAPPING, "%s: Writing DPM entry %d for target %d.\n", + __func__, mt_entry->dpm_entry_num, mt_entry->id); if (mpr_config_set_dpm_pg0(sc, &mpi_reply, &config_page, mt_entry->dpm_entry_num)) { - printf("%s: write of dpm entry %d for device failed\n", - __func__, mt_entry->dpm_entry_num); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: Write of DPM " + "entry %d for target %d failed.\n", __func__, + mt_entry->dpm_entry_num, mt_entry->id); dpm_entry->MappingInformation = le16toh(dpm_entry-> MappingInformation); dpm_entry->DeviceIndex = le16toh(dpm_entry->DeviceIndex); @@ -307,7 +325,7 @@ et_entry = &sc->enclosure_table[enc_idx]; if ((et_entry->missing_count > high_missing_count) && !et_entry->skip_search) { - high_missing_count = et_entry->missing_count; + high_missing_count = et_entry->missing_count; high_idx = enc_idx; } } @@ -326,7 +344,7 @@ static u32 _mapping_get_high_missing_mt_idx(struct mpr_softc *sc) { - u32 map_idx, high_idx = MPR_ENCTABLE_BAD_IDX; + u32 map_idx, high_idx = MPR_MAPTABLE_BAD_IDX; u8 high_missing_count = 0; u32 start_idx, end_idx, start_idx_ir, end_idx_ir; struct dev_mapping_table *mt_entry; @@ -370,7 +388,7 @@ _mapping_get_ir_maprange(sc, &start_idx, &end_idx); mt_entry = &sc->mapping_table[start_idx]; - for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) + for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) if (mt_entry->physical_id == wwid) return map_idx; @@ -458,20 +476,31 @@ u32 high_idx = MPR_MAPTABLE_BAD_IDX; struct dev_mapping_table *mt_entry; + /* + * The IN_USE flag should be clear if the entry is available to use. + * This flag is cleared on initialization and and when a volume is + * deleted. All other times this flag should be set. If, for some + * reason, a free entry cannot be found, look for the entry with the + * highest missing count just in case there is one. + */ _mapping_get_ir_maprange(sc, &start_idx, &end_idx); - mt_entry = &sc->mapping_table[start_idx]; - for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) + for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) { if (!(mt_entry->device_info & MPR_MAP_IN_USE)) return map_idx; - mt_entry = &sc->mapping_table[start_idx]; - for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) { if (mt_entry->missing_count > high_missing_count) { high_missing_count = mt_entry->missing_count; high_idx = map_idx; } } + + if (high_idx == MPR_MAPTABLE_BAD_IDX) { + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: Could not find a " + "free entry in the mapping table for a Volume. The mapping " + "table is probably corrupt.\n", __func__); + } + return high_idx; } @@ -494,6 +523,7 @@ if (sc->ir_firmware && (volume_mapping_flags == MPI2_IOCPAGE8_IRFLAGS_HIGH_VOLUME_MAPPING)) max_idx -= sc->max_volumes; + for (map_idx = start_idx; map_idx < max_idx; map_idx++, mt_entry++) if (!(mt_entry->device_info & (MPR_MAP_IN_USE | MPR_DEV_RESERVED))) @@ -542,12 +572,66 @@ _mapping_get_free_dpm_idx(struct mpr_softc *sc) { u16 entry_num; + Mpi2DriverMap0Entry_t *dpm_entry; + u16 current_entry = MPR_DPM_BAD_IDX, missing_cnt, high_missing_cnt = 0; + u64 physical_id; + struct dev_mapping_table *mt_entry; + u32 map_idx; for (entry_num = 0; entry_num < sc->max_dpm_entries; entry_num++) { - if (!sc->dpm_entry_used[entry_num]) - return entry_num; + dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); + dpm_entry += entry_num; + missing_cnt = dpm_entry->MappingInformation & + MPI2_DRVMAP0_MAPINFO_MISSING_MASK; + + /* + * If entry is used and not missing, then this entry can't be + * used. Look at next one. + */ + if (sc->dpm_entry_used[entry_num] && !missing_cnt) + continue; + + /* + * If this entry is not used at all, then the missing count + * doesn't matter. Just use this one. Otherwise, keep looking + * and make sure the entry with the highest missing count is + * used. + */ + if (!sc->dpm_entry_used[entry_num]) { + current_entry = entry_num; + break; + } + if ((current_entry == MPR_DPM_BAD_IDX) || + (missing_cnt > high_missing_cnt)) { + current_entry = entry_num; + high_missing_cnt = missing_cnt; + } } - return MPR_DPM_BAD_IDX; + + /* + * If an entry has been found to use and it's already marked as used + * it means that some device was already using this entry but it's + * missing, and that means that the connection between the missing + * device's DPM entry and the mapping table needs to be cleared. To do + * this, use the Physical ID of the old device still in the DPM entry + * to find its mapping table entry, then mark its DPM entry as BAD. + */ + if ((current_entry != MPR_DPM_BAD_IDX) && + sc->dpm_entry_used[current_entry]) { + dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); + dpm_entry += current_entry; + physical_id = dpm_entry->PhysicalIdentifier.High; + physical_id = (physical_id << 32) | + dpm_entry->PhysicalIdentifier.Low; + map_idx = _mapping_get_mt_idx_from_id(sc, physical_id); + if (map_idx != MPR_MAPTABLE_BAD_IDX) { + mt_entry = &sc->mapping_table[map_idx]; + mt_entry->dpm_entry_num = MPR_DPM_BAD_IDX; + } + } + return current_entry; } /** @@ -566,40 +650,57 @@ Mpi2EventIrConfigElement_t *element, u64 wwid) { struct dev_mapping_table *mt_entry; - u8 missing_cnt, reason = element->ReasonCode; + u8 missing_cnt, reason = element->ReasonCode, update_dpm = 1; u16 dpm_idx; Mpi2DriverMap0Entry_t *dpm_entry; - if (!sc->is_dpm_enable) - return; + /* + * Depending on the reason code, update the missing count. Always set + * the init_complete flag when here, so just do it first. That flag is + * used for volumes to make sure that the DPM entry has been updated. + * When a volume is deleted, clear the map entry's IN_USE flag so that + * the entry can be used again if another volume is created. Also clear + * its dev_handle entry so that other functions can't find this volume + * by the handle, since it's not defined any longer. + */ mt_entry = &sc->mapping_table[map_idx]; - if (reason == MPI2_EVENT_IR_CHANGE_RC_ADDED) { - mt_entry->missing_count = 0; - } else if (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED) { + mt_entry->init_complete = 1; + if ((reason == MPI2_EVENT_IR_CHANGE_RC_ADDED) || + (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED)) { mt_entry->missing_count = 0; - mt_entry->init_complete = 0; - } else if ((reason == MPI2_EVENT_IR_CHANGE_RC_REMOVED) || - (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED)) { - if (!mt_entry->init_complete) { - if (mt_entry->missing_count < MPR_MAX_MISSING_COUNT) - mt_entry->missing_count++; - else - mt_entry->init_complete = 1; - } - if (!mt_entry->missing_count) + } else if (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED) { + if (mt_entry->missing_count < MPR_MAX_MISSING_COUNT) mt_entry->missing_count++; + + mt_entry->device_info &= ~MPR_MAP_IN_USE; mt_entry->dev_handle = 0; } + /* + * If persistent mapping is enabled, update the DPM with the new missing + * count for the volume. If the DPM index is bad, get a free one. If + * it's bad for a volume that's being deleted do nothing because that + * volume doesn't have a DPM entry. + */ + if (!sc->is_dpm_enable) + return; dpm_idx = mt_entry->dpm_entry_num; if (dpm_idx == MPR_DPM_BAD_IDX) { - if ((reason == MPI2_EVENT_IR_CHANGE_RC_ADDED) || - (reason == MPI2_EVENT_IR_CHANGE_RC_REMOVED)) - dpm_idx = _mapping_get_dpm_idx_from_id(sc, - mt_entry->physical_id, 0); - else if (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED) + if (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED) + { + mpr_dprint(sc, MPR_MAPPING, "%s: Volume being deleted " + "is not in DPM so DPM missing count will not be " + "updated.\n", __func__); return; + } } + if (dpm_idx == MPR_DPM_BAD_IDX) + dpm_idx = _mapping_get_free_dpm_idx(sc); + + /* + * Got the DPM entry for the volume or found a free DPM entry if this is + * a new volume. Check if the current information is outdated. + */ if (dpm_idx != MPR_DPM_BAD_IDX) { dpm_entry = (Mpi2DriverMap0Entry_t *)((u8 *)sc->dpm_pg0 + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); @@ -607,17 +708,24 @@ missing_cnt = dpm_entry->MappingInformation & MPI2_DRVMAP0_MAPINFO_MISSING_MASK; if ((mt_entry->physical_id == - le64toh((u64)dpm_entry->PhysicalIdentifier.High | - dpm_entry->PhysicalIdentifier.Low)) && (missing_cnt == - mt_entry->missing_count)) - mt_entry->init_complete = 1; - } else { - dpm_idx = _mapping_get_free_dpm_idx(sc); - mt_entry->init_complete = 0; + le64toh(((u64)dpm_entry->PhysicalIdentifier.High << 32) | + (u64)dpm_entry->PhysicalIdentifier.Low)) && (missing_cnt == + mt_entry->missing_count)) { + mpr_dprint(sc, MPR_MAPPING, "%s: DPM entry for volume " + "with target ID %d does not require an update.\n", + __func__, mt_entry->id); + update_dpm = 0; + } } - if ((dpm_idx != MPR_DPM_BAD_IDX) && !mt_entry->init_complete) { - mt_entry->init_complete = 1; + /* + * Update the volume's persistent info if it's new or the ID or missing + * count has changed. If a good DPM index has not been found by now, + * there is no space left in the DPM table. + */ + if ((dpm_idx != MPR_DPM_BAD_IDX) && update_dpm) { + mpr_dprint(sc, MPR_MAPPING, "%s: Update DPM entry for volume " + "with target ID %d.\n", __func__, mt_entry->id); mt_entry->dpm_entry_num = dpm_idx; dpm_entry = (Mpi2DriverMap0Entry_t *)((u8 *)sc->dpm_pg0 + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); @@ -633,44 +741,46 @@ sc->dpm_flush_entry[dpm_idx] = 1; sc->dpm_entry_used[dpm_idx] = 1; } else if (dpm_idx == MPR_DPM_BAD_IDX) { - printf("%s: no space to add entry in DPM table\n", __func__); - mt_entry->init_complete = 1; + mpr_dprint(sc, MPR_INFO | MPR_MAPPING, "%s: No space to add an " + "entry in the DPM table for volume with target ID %d.\n", + __func__, mt_entry->id); } } /** - * _mapping_add_to_removal_table - mark an entry for removal + * _mapping_add_to_removal_table - add DPM index to the removal table * @sc: per adapter object - * @handle: Handle of enclosures/device/volume + * @dpm_idx: Index of DPM entry to remove * - * Adds the handle or DPM entry number in removal table. + * Adds a DPM entry number to the removal table. * * Returns nothing. */ static void -_mapping_add_to_removal_table(struct mpr_softc *sc, u16 handle, - u16 dpm_idx) +_mapping_add_to_removal_table(struct mpr_softc *sc, u16 dpm_idx) { struct map_removal_table *remove_entry; u32 i; - u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags); - remove_entry = sc->removal_table; + /* + * This is only used to remove entries from the DPM in the controller. + * If DPM is not enabled, just return. + */ + if (!sc->is_dpm_enable) + return; + /* + * Find the first available removal_table entry and add the new entry + * there. + */ + remove_entry = sc->removal_table; for (i = 0; i < sc->max_devices; i++, remove_entry++) { - if (remove_entry->dev_handle || remove_entry->dpm_entry_num != - MPR_DPM_BAD_IDX) + if (remove_entry->dpm_entry_num != MPR_DPM_BAD_IDX) continue; - if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == - MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { - if (dpm_idx) - remove_entry->dpm_entry_num = dpm_idx; - if (remove_entry->dpm_entry_num == MPR_DPM_BAD_IDX) - remove_entry->dev_handle = handle; - } else if ((ioc_pg8_flags & - MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == - MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) - remove_entry->dev_handle = handle; + + mpr_dprint(sc, MPR_MAPPING, "%s: Adding DPM entry %d to table " + "for removal.\n", __func__, dpm_idx); + remove_entry->dpm_entry_num = dpm_idx; break; } @@ -683,8 +793,11 @@ * * Increment the missing count in the mapping table for a SAS, SATA, or PCIe * device that is not responding. If Persitent Mapping is used, increment the - * DPM entry as well. Also, add this device to the removal table for possible - * removal if a new device is added. + * DPM entry as well. Currently, this function is only called if the target + * goes missing, so after initialization has completed. This means that the + * missing count can only go from 0 to 1 here. The missing count is incremented + * during initialization as well, so that's where a target's missing count can + * go past 1. * * Returns nothing. */ @@ -696,33 +809,40 @@ Mpi2DriverMap0Entry_t *dpm_entry; if (map_idx == MPR_MAPTABLE_BAD_IDX) { - mpr_dprint(sc, MPR_INFO, "%s: device is already removed from " - "mapping table\n", __func__); + mpr_dprint(sc, MPR_INFO | MPR_MAPPING, "%s: device is already " + "removed from mapping table\n", __func__); return; } mt_entry = &sc->mapping_table[map_idx]; - if (!mt_entry->init_complete) { - if (mt_entry->missing_count < MPR_MAX_MISSING_COUNT) - mt_entry->missing_count++; - else - mt_entry->init_complete = 1; - } - if (!mt_entry->missing_count) + if (mt_entry->missing_count < MPR_MAX_MISSING_COUNT) mt_entry->missing_count++; - _mapping_add_to_removal_table(sc, mt_entry->dev_handle, 0); - mt_entry->dev_handle = 0; + /* + * When using Enc/Slot mapping, when a device is removed, it's mapping + * table information should be cleared. Otherwise, the target ID will + * be incorrect if this same device is re-added to a different slot. + */ + if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == + MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { + _mapping_clear_map_entry(mt_entry); + } + + /* + * When using device mapping, update the missing count in the DPM entry, + * but only if the missing count has changed. + */ if (((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) && - sc->is_dpm_enable && !mt_entry->init_complete && + sc->is_dpm_enable && mt_entry->dpm_entry_num != MPR_DPM_BAD_IDX) { dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); dpm_entry += mt_entry->dpm_entry_num; - dpm_entry->MappingInformation = mt_entry->missing_count; - sc->dpm_flush_entry[mt_entry->dpm_entry_num] = 1; + if (dpm_entry->MappingInformation != mt_entry->missing_count) { + dpm_entry->MappingInformation = mt_entry->missing_count; + sc->dpm_flush_entry[mt_entry->dpm_entry_num] = 1; + } } - mt_entry->init_complete = 1; } /** @@ -814,6 +934,10 @@ vol_mapping_flags = le16toh(sc->ioc_pg8.IRVolumeMappingFlags) & MPI2_IOCPAGE8_IRFLAGS_MASK_VOLUME_MAPPING_MODE; + /* + * The end of the mapping table depends on where volumes are kept, if + * IR is enabled. + */ if (!sc->ir_firmware) end_of_table = sc->max_devices; else if (vol_mapping_flags == MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING) @@ -821,6 +945,17 @@ else end_of_table = sc->max_devices - sc->max_volumes; + /* + * The skip_count is the number of entries that are reserved at the + * beginning of the mapping table. But, it does not include the number + * of Physical IDs that are reserved for direct attached devices. Look + * through the mapping table after these reserved entries to see if + * the devices for this enclosure are already mapped. The PHY bit check + * is used to make sure that at least one PHY bit is common between the + * enclosure and the device that is already mapped. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Looking for space in the mapping " + "table for added enclosure.\n", __func__); for (map_idx = (max_num_phy_ids + skip_count); map_idx < end_of_table; map_idx++) { mt_entry = &sc->mapping_table[map_idx]; @@ -830,11 +965,21 @@ num_found += 1; if (num_found == et_entry->num_slots) { start_idx = (map_idx - num_found) + 1; + mpr_dprint(sc, MPR_MAPPING, "%s: Found space " + "in the mapping for enclosure at map index " + "%d.\n", __func__, start_idx); return start_idx; } } else num_found = 0; } + + /* + * If the enclosure's devices are not mapped already, look for + * contiguous entries in the mapping table that are not reserved. If + * enough entries are found, return the starting index for that space. + */ + num_found = 0; for (map_idx = (max_num_phy_ids + skip_count); map_idx < end_of_table; map_idx++) { mt_entry = &sc->mapping_table[map_idx]; @@ -842,40 +987,91 @@ num_found += 1; if (num_found == et_entry->num_slots) { start_idx = (map_idx - num_found) + 1; + mpr_dprint(sc, MPR_MAPPING, "%s: Found space " + "in the mapping for enclosure at map index " + "%d.\n", __func__, start_idx); return start_idx; } } else num_found = 0; } + /* + * If here, it means that not enough space in the mapping table was + * found to support this enclosure, so go through the enclosure table to + * see if any enclosure entries have a missing count. If so, get the + * enclosure with the highest missing count and check it to see if there + * is enough space for the new enclosure. + */ while (!done_flag) { enc_idx = _mapping_get_high_missing_et_idx(sc); - if (enc_idx == MPR_ENCTABLE_BAD_IDX) + if (enc_idx == MPR_ENCTABLE_BAD_IDX) { + mpr_dprint(sc, MPR_MAPPING, "%s: Not enough space was " + "found in the mapping for the added enclosure.\n", + __func__); return MPR_MAPTABLE_BAD_IDX; + } + + /* + * Found a missing enclosure. Set the skip_search flag so this + * enclosure is not checked again for a high missing count if + * the loop continues. This way, all missing enclosures can + * have their space added together to find enough space in the + * mapping table for the added enclosure. The space must be + * contiguous. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Space from a missing " + "enclosure was found.\n", __func__); enc_entry = &sc->enclosure_table[enc_idx]; - /*VSP FIXME*/ enc_entry->skip_search = 1; + + /* + * Unmark all of the missing enclosure's device's reserved + * space. These will be remarked as reserved if this missing + * enclosure's space is not used. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Clear the reserved flag for " + "all of the map entries for the enclosure.\n", __func__); mt_entry = &sc->mapping_table[enc_entry->start_index]; for (map_idx = enc_entry->start_index; map_idx < (enc_entry->start_index + enc_entry->num_slots); map_idx++, mt_entry++) - mt_entry->device_info &= ~MPR_DEV_RESERVED; + mt_entry->device_info &= ~MPR_DEV_RESERVED; + + /* + * Now that space has been unreserved, check again to see if + * enough space is available for the new enclosure. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Check if new mapping space is " + "enough for the new enclosure.\n", __func__); found_space = 0; - for (map_idx = (max_num_phy_ids + - skip_count); map_idx < end_of_table; map_idx++) { + num_found = 0; + for (map_idx = (max_num_phy_ids + skip_count); + map_idx < end_of_table; map_idx++) { mt_entry = &sc->mapping_table[map_idx]; if (!(mt_entry->device_info & MPR_DEV_RESERVED)) { num_found += 1; if (num_found == et_entry->num_slots) { start_idx = (map_idx - num_found) + 1; found_space = 1; + break; } } else num_found = 0; } - if (!found_space) continue; + + /* + * If enough space was found, all of the missing enclosures that + * will be used for the new enclosure must be added to the + * removal table. Then all mappings for the enclosure's devices + * and for the enclosure itself need to be cleared. There may be + * more than one enclosure to add to the removal table and + * clear. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Found space in the mapping " + "for enclosure at map index %d.\n", __func__, start_idx); for (map_idx = start_idx; map_idx < (start_idx + num_found); map_idx++) { enc_entry = sc->enclosure_table; @@ -886,26 +1082,38 @@ enc_entry->num_slots)) continue; if (!enc_entry->removal_flag) { + mpr_dprint(sc, MPR_MAPPING, "%s: " + "Enclosure %d will be removed from " + "the mapping table.\n", __func__, + enc_idx); enc_entry->removal_flag = 1; - _mapping_add_to_removal_table(sc, 0, + _mapping_add_to_removal_table(sc, enc_entry->dpm_entry_num); } mt_entry = &sc->mapping_table[map_idx]; - if (mt_entry->device_info & - MPR_MAP_IN_USE) { - _mapping_add_to_removal_table(sc, - mt_entry->dev_handle, 0); - _mapping_clear_map_entry(mt_entry); - } + _mapping_clear_map_entry(mt_entry); if (map_idx == (enc_entry->start_index + enc_entry->num_slots - 1)) _mapping_clear_enc_entry(et_entry); } } + + /* + * During the search for space for this enclosure, some entries + * in the mapping table may have been unreserved. Go back and + * change all of these to reserved again. Only the enclosures + * with the removal_flag set should be left as unreserved. The + * skip_search flag needs to be cleared as well so that the + * enclosure's space will be looked at the next time space is + * needed. + */ enc_entry = sc->enclosure_table; for (enc_idx = 0; enc_idx < sc->num_enc_table_entries; enc_idx++, enc_entry++) { if (!enc_entry->removal_flag) { + mpr_dprint(sc, MPR_MAPPING, "%s: Reset the " + "reserved flag for all of the map entries " + "for enclosure %d.\n", __func__, enc_idx); mt_entry = &sc->mapping_table[enc_entry-> start_index]; for (map_idx = enc_entry->start_index; map_idx < @@ -939,7 +1147,7 @@ u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags); Mpi2ConfigReply_t mpi_reply; Mpi2SasDevicePage0_t sas_device_pg0; - u8 entry, enc_idx, phy_idx, sata_end_device; + u8 entry, enc_idx, phy_idx; u32 map_idx, index, device_info; struct _map_phy_change *phy_change, *tmp_phy_change; uint64_t sas_address; @@ -953,6 +1161,7 @@ if (phy_change->is_processed || !phy_change->dev_handle || phy_change->reason != MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED) continue; + if (mpr_config_get_sas_device_pg0(sc, &mpi_reply, &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, phy_change->dev_handle)) { @@ -966,13 +1175,11 @@ * when the system is shutdown. */ device_info = le32toh(sas_device_pg0.DeviceInfo); - sas_address = sas_device_pg0.SASAddress.High; + sas_address = le32toh(sas_device_pg0.SASAddress.High); sas_address = (sas_address << 32) | - sas_device_pg0.SASAddress.Low; - sata_end_device = 0; + le32toh(sas_device_pg0.SASAddress.Low); if ((device_info & MPI2_SAS_DEVICE_INFO_END_DEVICE) && (device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)) { - sata_end_device = 1; rc = mprsas_get_sas_address_for_sata_disk(sc, &sas_address, phy_change->dev_handle, device_info, &phy_change->is_SATA_SSD); @@ -991,16 +1198,27 @@ phy_change->slot = le16toh(sas_device_pg0.Slot); phy_change->device_info = device_info; + /* + * When using Enc/Slot mapping, if this device is an enclosure + * make sure that all of its slots can fit into the mapping + * table. + */ if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { + /* + * The enclosure should already be in the enclosure + * table due to the Enclosure Add event. If not, just + * continue, nothing can be done. + */ enc_idx = _mapping_get_enc_idx_from_handle(sc, topo_change->enc_handle); if (enc_idx == MPR_ENCTABLE_BAD_IDX) { phy_change->is_processed = 1; - mpr_dprint(sc, MPR_MAPPING, "%s: failed to add " - "the device with handle 0x%04x because the " - "enclosure is not in the mapping table\n", - __func__, phy_change->dev_handle); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because the enclosure is not in " + "the mapping table\n", __func__, + phy_change->dev_handle); continue; } if (!((phy_change->device_info & @@ -1013,8 +1231,20 @@ continue; } et_entry = &sc->enclosure_table[enc_idx]; + + /* + * If the enclosure already has a start_index, it's been + * mapped, so go to the next Topo change. + */ if (et_entry->start_index != MPR_MAPTABLE_BAD_IDX) continue; + + /* + * If the Expander Handle is 0, the devices are direct + * attached. In that case, the start_index must be just + * after the reserved entries. Otherwise, find space in + * the mapping table for the enclosure's devices. + */ if (!topo_change->exp_handle) { map_idx = sc->num_rsvd_entries; et_entry->start_index = map_idx; @@ -1022,8 +1252,26 @@ map_idx = _mapping_find_enc_map_space(sc, et_entry); et_entry->start_index = map_idx; + + /* + * If space cannot be found to hold all of the + * enclosure's devices in the mapping table, + * there's no need to continue checking the + * other devices in this event. Set all of the + * phy_details for this event (if the change is + * for an add) as already processed because none + * of these devices can be added to the mapping + * table. + */ if (et_entry->start_index == MPR_MAPTABLE_BAD_IDX) { + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, + "%s: failed to add the enclosure " + "with ID 0x%016jx because there is " + "no free space available in the " + "mapping table for all of the " + "enclosure's devices.\n", __func__, + (uintmax_t)et_entry->enclosure_id); phy_change->is_processed = 1; for (phy_idx = 0; phy_idx < topo_change->num_entries; @@ -1040,7 +1288,14 @@ } } - /* Found space in enclosure for mapping entry */ + /* + * Found space in the mapping table for this enclosure. + * Initialize each mapping table entry for the + * enclosure. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Initialize %d map " + "entries for the enclosure, starting at map index " + " %d.\n", __func__, et_entry->num_slots, map_idx); mt_entry = &sc->mapping_table[map_idx]; for (index = map_idx; index < (et_entry->num_slots + map_idx); index++, mt_entry++) { @@ -1098,16 +1353,27 @@ port_change->slot = le16toh(pcie_device_pg0.Slot); port_change->device_info = le32toh(pcie_device_pg0.DeviceInfo); + /* + * When using Enc/Slot mapping, if this device is an enclosure + * make sure that all of its slots can fit into the mapping + * table. + */ if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { + /* + * The enclosure should already be in the enclosure + * table due to the Enclosure Add event. If not, just + * continue, nothing can be done. + */ enc_idx = _mapping_get_enc_idx_from_handle(sc, topo_change->enc_handle); if (enc_idx == MPR_ENCTABLE_BAD_IDX) { port_change->is_processed = 1; - mpr_dprint(sc, MPR_MAPPING, "%s: failed to add " - "the device with handle 0x%04x because the " - "enclosure is not in the mapping table\n", - __func__, port_change->dev_handle); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because the enclosure is not in " + "the mapping table\n", __func__, + port_change->dev_handle); continue; } if (!(port_change->device_info & @@ -1116,8 +1382,20 @@ continue; } et_entry = &sc->enclosure_table[enc_idx]; + + /* + * If the enclosure already has a start_index, it's been + * mapped, so go to the next Topo change. + */ if (et_entry->start_index != MPR_MAPTABLE_BAD_IDX) continue; + + /* + * If the Switch Handle is 0, the devices are direct + * attached. In that case, the start_index must be just + * after the reserved entries. Otherwise, find space in + * the mapping table for the enclosure's devices. + */ if (!topo_change->switch_dev_handle) { map_idx = sc->num_rsvd_entries; et_entry->start_index = map_idx; @@ -1125,8 +1403,26 @@ map_idx = _mapping_find_enc_map_space(sc, et_entry); et_entry->start_index = map_idx; + + /* + * If space cannot be found to hold all of the + * enclosure's devices in the mapping table, + * there's no need to continue checking the + * other devices in this event. Set all of the + * port_details for this event (if the change is + * for an add) as already processed because none + * of these devices can be added to the mapping + * table. + */ if (et_entry->start_index == MPR_MAPTABLE_BAD_IDX) { + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, + "%s: failed to add the enclosure " + "with ID 0x%016jx because there is " + "no free space available in the " + "mapping table for all of the " + "enclosure's devices.\n", __func__, + (uintmax_t)et_entry->enclosure_id); port_change->is_processed = 1; for (port_idx = 0; port_idx < topo_change->num_entries; @@ -1143,7 +1439,14 @@ } } - /* Found space in enclosure for mapping entry */ + /* + * Found space in the mapping table for this enclosure. + * Initialize each mapping table entry for the + * enclosure. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Initialize %d map " + "entries for the enclosure, starting at map index " + " %d.\n", __func__, et_entry->num_slots, map_idx); mt_entry = &sc->mapping_table[map_idx]; for (index = map_idx; index < (et_entry->num_slots + map_idx); index++, mt_entry++) { @@ -1170,6 +1473,7 @@ struct dev_mapping_table *mt_entry; u16 slots = et_entry->num_slots, map_idx; u32 start_idx = et_entry->start_index; + if (start_idx != MPR_MAPTABLE_BAD_IDX) { mt_entry = &sc->mapping_table[start_idx]; for (map_idx = 0; map_idx < slots; map_idx++, mt_entry++) @@ -1219,6 +1523,13 @@ } } } + + /* + * When using Enc/Slot mapping, if a new enclosure was added and old + * enclosure space was needed, the enclosure table may now have gaps + * that need to be closed. All enclosure mappings need to be contiguous + * so that space can be reused correctly if available. + */ if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { num_entries = sc->num_enc_table_entries; @@ -1299,31 +1610,41 @@ (sc, topo_change->enc_handle); if (enc_idx == MPR_ENCTABLE_BAD_IDX) { phy_change->is_processed = 1; - mpr_dprint(sc, MPR_ERROR, "%s: failed to add " - "the device with handle 0x%04x because the " - "enclosure is not in the mapping table\n", - __func__, phy_change->dev_handle); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because the enclosure is not in " + "the mapping table\n", __func__, + phy_change->dev_handle); continue; } + + /* + * If the enclosure's start_index is BAD here, it means + * that there is no room in the mapping table to cover + * all of the devices that could be in the enclosure. + * There's no reason to process any of the devices for + * this enclosure since they can't be mapped. + */ et_entry = &sc->enclosure_table[enc_idx]; if (et_entry->start_index == MPR_MAPTABLE_BAD_IDX) { phy_change->is_processed = 1; - if (!sc->mt_full_retry) { - sc->mt_add_device_failed = 1; - continue; - } - mpr_dprint(sc, MPR_INFO, "%s: failed to add " - "the device with handle 0x%04x because " - "there is no free space available in the " - "mapping table\n", __func__, - phy_change->dev_handle); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because there is no free space " + "available in the mapping table\n", + __func__, phy_change->dev_handle); continue; } + + /* + * Add this device to the mapping table at the correct + * offset where space was found to map the enclosure. + * Then setup the DPM entry information if being used. + */ map_idx = et_entry->start_index + phy_change->slot - et_entry->start_slot; mt_entry = &sc->mapping_table[map_idx]; mt_entry->physical_id = phy_change->physical_id; - mt_entry->channel = 0; mt_entry->id = map_idx; mt_entry->dev_handle = phy_change->dev_handle; mt_entry->missing_count = 0; @@ -1350,28 +1671,28 @@ et_entry->enclosure_id); dpm_entry-> PhysicalIdentifier.High = - ( et_entry->enclosure_id + (et_entry->enclosure_id >> 32); dpm_entry->DeviceIndex = (U16)et_entry->start_index; dpm_entry->MappingInformation = - et_entry->num_slots; + et_entry->num_slots; dpm_entry->MappingInformation <<= map_shift; dpm_entry->PhysicalBitsMapping = et_entry->phy_bits; et_entry->dpm_entry_num = dpm_idx; - /* FIXME Do I need to set the dpm_idxin mt_entry too */ sc->dpm_entry_used[dpm_idx] = 1; sc->dpm_flush_entry[dpm_idx] = 1; phy_change->is_processed = 1; } else { phy_change->is_processed = 1; - mpr_dprint(sc, MPR_INFO, "%s: " - "failed to add the device " - "with handle 0x%04x to " + mpr_dprint(sc, MPR_ERROR | + MPR_MAPPING, "%s: failed " + "to add the device with " + "handle 0x%04x to " "persistent table because " "there is no free space " "available\n", __func__, @@ -1382,11 +1703,20 @@ mt_entry->dpm_entry_num = dpm_idx; } } - /* FIXME Why not mt_entry too? */ et_entry->init_complete = 1; } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + + /* + * Get the mapping table index for this device. If it's + * not in the mapping table yet, find a free entry if + * one is available. If there are no free entries, look + * for the entry that has the highest missing count. If + * none of that works to find an entry in the mapping + * table, there is a problem. Log a message and just + * continue on. + */ map_idx = _mapping_get_mt_idx_from_id (sc, phy_change->physical_id); if (map_idx == MPR_MAPTABLE_BAD_IDX) { @@ -1396,23 +1726,24 @@ map_idx = _mapping_get_free_mt_idx(sc, search_idx); } + + /* + * If an entry will be used that has a missing device, + * clear its entry from the DPM in the controller. + */ if (map_idx == MPR_MAPTABLE_BAD_IDX) { map_idx = _mapping_get_high_missing_mt_idx(sc); if (map_idx != MPR_MAPTABLE_BAD_IDX) { mt_entry = &sc->mapping_table[map_idx]; - if (mt_entry->dev_handle) { - _mapping_add_to_removal_table - (sc, mt_entry->dev_handle, - 0); - is_removed = 1; - } + _mapping_add_to_removal_table(sc, + mt_entry->dpm_entry_num); + is_removed = 1; mt_entry->init_complete = 0; } } if (map_idx != MPR_MAPTABLE_BAD_IDX) { mt_entry = &sc->mapping_table[map_idx]; mt_entry->physical_id = phy_change->physical_id; - mt_entry->channel = 0; mt_entry->id = map_idx; mt_entry->dev_handle = phy_change->dev_handle; mt_entry->missing_count = 0; @@ -1420,15 +1751,11 @@ | (MPR_DEV_RESERVED | MPR_MAP_IN_USE); } else { phy_change->is_processed = 1; - if (!sc->mt_full_retry) { - sc->mt_add_device_failed = 1; - continue; - } - mpr_dprint(sc, MPR_INFO, "%s: failed to add " - "the device with handle 0x%04x because " - "there is no free space available in the " - "mapping table\n", __func__, - phy_change->dev_handle); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because there is no free space " + "available in the mapping table\n", + __func__, phy_change->dev_handle); continue; } if (sc->is_dpm_enable) { @@ -1445,16 +1772,24 @@ PhysicalIdentifier.High; temp64_var = (temp64_var << 32) | dpm_entry->PhysicalIdentifier.Low; + + /* + * If the Mapping Table's info is not + * the same as the DPM entry, clear the + * init_complete flag so that it's + * updated. + */ if ((mt_entry->physical_id == temp64_var) && !missing_cnt) mt_entry->init_complete = 1; + else + mt_entry->init_complete = 0; } else { dpm_idx = _mapping_get_free_dpm_idx(sc); mt_entry->init_complete = 0; } if (dpm_idx != MPR_DPM_BAD_IDX && !mt_entry->init_complete) { - mt_entry->init_complete = 1; mt_entry->dpm_entry_num = dpm_idx; dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + hdr_sz); @@ -1472,9 +1807,9 @@ phy_change->is_processed = 1; } else if (dpm_idx == MPR_DPM_BAD_IDX) { phy_change->is_processed = 1; - mpr_dprint(sc, MPR_INFO, "%s: failed " - "to add the device with handle " - "0x%04x to persistent table " + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, + "%s: failed to add the device with " + "handle 0x%04x to persistent table " "because there is no free space " "available\n", __func__, phy_change->dev_handle); @@ -1532,31 +1867,41 @@ (sc, topo_change->enc_handle); if (enc_idx == MPR_ENCTABLE_BAD_IDX) { port_change->is_processed = 1; - mpr_dprint(sc, MPR_ERROR, "%s: failed to add " - "the device with handle 0x%04x because the " - "enclosure is not in the mapping table\n", - __func__, port_change->dev_handle); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because the enclosure is not in " + "the mapping table\n", __func__, + port_change->dev_handle); continue; } + + /* + * If the enclosure's start_index is BAD here, it means + * that there is no room in the mapping table to cover + * all of the devices that could be in the enclosure. + * There's no reason to process any of the devices for + * this enclosure since they can't be mapped. + */ et_entry = &sc->enclosure_table[enc_idx]; if (et_entry->start_index == MPR_MAPTABLE_BAD_IDX) { port_change->is_processed = 1; - if (!sc->mt_full_retry) { - sc->mt_add_device_failed = 1; - continue; - } - mpr_dprint(sc, MPR_INFO, "%s: failed to add " - "the device with handle 0x%04x because " - "there is no free space available in the " - "mapping table\n", __func__, - port_change->dev_handle); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because there is no free space " + "available in the mapping table\n", + __func__, port_change->dev_handle); continue; } + + /* + * Add this device to the mapping table at the correct + * offset where space was found to map the enclosure. + * Then setup the DPM entry information if being used. + */ map_idx = et_entry->start_index + port_change->slot - et_entry->start_slot; mt_entry = &sc->mapping_table[map_idx]; mt_entry->physical_id = port_change->physical_id; - mt_entry->channel = 0; mt_entry->id = map_idx; mt_entry->dev_handle = port_change->dev_handle; mt_entry->missing_count = 0; @@ -1583,28 +1928,28 @@ et_entry->enclosure_id); dpm_entry-> PhysicalIdentifier.High = - ( et_entry->enclosure_id + (et_entry->enclosure_id >> 32); dpm_entry->DeviceIndex = (U16)et_entry->start_index; dpm_entry->MappingInformation = - et_entry->num_slots; + et_entry->num_slots; dpm_entry->MappingInformation <<= map_shift; dpm_entry->PhysicalBitsMapping = et_entry->phy_bits; et_entry->dpm_entry_num = dpm_idx; - /* FIXME Do I need to set the dpm_idxin mt_entry too */ sc->dpm_entry_used[dpm_idx] = 1; sc->dpm_flush_entry[dpm_idx] = 1; port_change->is_processed = 1; } else { port_change->is_processed = 1; - mpr_dprint(sc, MPR_INFO, "%s: " - "failed to add the device " - "with handle 0x%04x to " + mpr_dprint(sc, MPR_ERROR | + MPR_MAPPING, "%s: failed " + "to add the device with " + "handle 0x%04x to " "persistent table because " "there is no free space " "available\n", __func__, @@ -1615,11 +1960,20 @@ mt_entry->dpm_entry_num = dpm_idx; } } - /* FIXME Why not mt_entry too? */ et_entry->init_complete = 1; } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + + /* + * Get the mapping table index for this device. If it's + * not in the mapping table yet, find a free entry if + * one is available. If there are no free entries, look + * for the entry that has the highest missing count. If + * none of that works to find an entry in the mapping + * table, there is a problem. Log a message and just + * continue on. + */ map_idx = _mapping_get_mt_idx_from_id (sc, port_change->physical_id); if (map_idx == MPR_MAPTABLE_BAD_IDX) { @@ -1629,16 +1983,18 @@ map_idx = _mapping_get_free_mt_idx(sc, search_idx); } + + /* + * If an entry will be used that has a missing device, + * clear its entry from the DPM in the controller. + */ if (map_idx == MPR_MAPTABLE_BAD_IDX) { map_idx = _mapping_get_high_missing_mt_idx(sc); if (map_idx != MPR_MAPTABLE_BAD_IDX) { mt_entry = &sc->mapping_table[map_idx]; - if (mt_entry->dev_handle) { - _mapping_add_to_removal_table - (sc, mt_entry->dev_handle, - 0); - is_removed = 1; - } + _mapping_add_to_removal_table(sc, + mt_entry->dpm_entry_num); + is_removed = 1; mt_entry->init_complete = 0; } } @@ -1646,7 +2002,6 @@ mt_entry = &sc->mapping_table[map_idx]; mt_entry->physical_id = port_change->physical_id; - mt_entry->channel = 0; mt_entry->id = map_idx; mt_entry->dev_handle = port_change->dev_handle; mt_entry->missing_count = 0; @@ -1655,15 +2010,11 @@ (MPR_DEV_RESERVED | MPR_MAP_IN_USE); } else { port_change->is_processed = 1; - if (!sc->mt_full_retry) { - sc->mt_add_device_failed = 1; - continue; - } - mpr_dprint(sc, MPR_INFO, "%s: failed to add " - "the device with handle 0x%04x because " - "there is no free space available in the " - "mapping table\n", __func__, - port_change->dev_handle); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because there is no free space " + "available in the mapping table\n", + __func__, port_change->dev_handle); continue; } if (sc->is_dpm_enable) { @@ -1680,16 +2031,24 @@ PhysicalIdentifier.High; temp64_var = (temp64_var << 32) | dpm_entry->PhysicalIdentifier.Low; + + /* + * If the Mapping Table's info is not + * the same as the DPM entry, clear the + * init_complete flag so that it's + * updated. + */ if ((mt_entry->physical_id == temp64_var) && !missing_cnt) mt_entry->init_complete = 1; + else + mt_entry->init_complete = 0; } else { dpm_idx = _mapping_get_free_dpm_idx(sc); mt_entry->init_complete = 0; } if (dpm_idx != MPR_DPM_BAD_IDX && !mt_entry->init_complete) { - mt_entry->init_complete = 1; mt_entry->dpm_entry_num = dpm_idx; dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + hdr_sz); @@ -1707,9 +2066,9 @@ port_change->is_processed = 1; } else if (dpm_idx == MPR_DPM_BAD_IDX) { port_change->is_processed = 1; - mpr_dprint(sc, MPR_INFO, "%s: failed " - "to add the device with handle " - "0x%04x to persistent table " + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, + "%s: failed to add the device with " + "handle 0x%04x to persistent table " "because there is no free space " "available\n", __func__, port_change->dev_handle); @@ -1755,10 +2114,13 @@ memcpy(&config_page.Entry, (u8 *)dpm_entry, sizeof(Mpi2DriverMap0Entry_t)); /* TODO-How to handle failed writes? */ + mpr_dprint(sc, MPR_MAPPING, "%s: Flushing DPM entry %d.\n", + __func__, entry_num); if (mpr_config_set_dpm_pg0(sc, &mpi_reply, &config_page, entry_num)) { - printf("%s: write of dpm entry %d for device failed\n", - __func__, entry_num); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: Flush of " + "DPM entry %d for device failed\n", __func__, + entry_num); } else sc->dpm_flush_entry[entry_num] = 0; dpm_entry->MappingInformation = le16toh(dpm_entry-> @@ -1848,7 +2210,6 @@ free(sc->dpm_pg0, M_MPR); } - static void _mapping_process_dpm_pg0(struct mpr_softc *sc) { @@ -1863,9 +2224,20 @@ u64 physical_id; u32 phy_bits = 0; + /* + * start_idx and end_idx are only used for IR. + */ if (sc->ir_firmware) _mapping_get_ir_maprange(sc, &start_idx, &end_idx); + /* + * Look through all of the DPM entries that were read from the + * controller and copy them over to the driver's internal table if they + * have a non-zero ID. At this point, any ID with a value of 0 would be + * invalid, so don't copy it. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Start copy of %d DPM entries into the " + "mapping table.\n", __func__, sc->max_dpm_entries); dpm_entry = (Mpi2DriverMap0Entry_t *) ((uint8_t *) sc->dpm_pg0 + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); for (entry_num = 0; entry_num < sc->max_dpm_entries; entry_num++, @@ -1884,13 +2256,20 @@ MPI2_DRVMAP0_MAPINFO_MISSING_MASK; dev_idx = le16toh(dpm_entry->DeviceIndex); phy_bits = le32toh(dpm_entry->PhysicalBitsMapping); + + /* + * Volumes are at special locations in the mapping table so + * account for that. Volume mapping table entries do not depend + * on the type of mapping, so continue the loop after adding + * volumes to the mapping table. + */ if (sc->ir_firmware && (dev_idx >= start_idx) && (dev_idx <= end_idx)) { mt_entry = &sc->mapping_table[dev_idx]; - mt_entry->physical_id = dpm_entry->PhysicalIdentifier.High; + mt_entry->physical_id = + dpm_entry->PhysicalIdentifier.High; mt_entry->physical_id = (mt_entry->physical_id << 32) | dpm_entry->PhysicalIdentifier.Low; - mt_entry->channel = MPR_RAID_CHANNEL; mt_entry->id = dev_idx; mt_entry->missing_count = missing_cnt; mt_entry->dpm_entry_num = entry_num; @@ -1899,7 +2278,16 @@ } if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { - if (dev_idx < (sc->num_rsvd_entries + + + /* + * The dev_idx for an enclosure is the start index. If + * the start index is within the controller's default + * enclosure area, set the number of slots for this + * enclosure to the max allowed. Otherwise, it should be + * a normal enclosure and the number of slots is in the + * DPM entry's Mapping Information. + */ + if (dev_idx < (sc->num_rsvd_entries + max_num_phy_ids)) { slot_id = 0; if (ioc_pg8_flags & @@ -1914,8 +2302,9 @@ } enc_idx = sc->num_enc_table_entries; if (enc_idx >= sc->max_enclosures) { - printf("%s: enclosure entries exceed max " - "enclosures of %d\n", __func__, + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "Number of enclosure entries in DPM exceed " + "the max allowed of %d.\n", __func__, sc->max_enclosures); break; } @@ -1931,21 +2320,32 @@ et_entry->missing_count = missing_cnt; et_entry->phy_bits = phy_bits; + /* + * Initialize all entries for this enclosure in the + * mapping table and mark them as reserved. The actual + * devices have not been processed yet but when they are + * they will use these entries. If an entry is found + * that already has a valid DPM index, the mapping table + * is corrupt. This can happen if the mapping type is + * changed without clearing all of the DPM entries in + * the controller. + */ mt_entry = &sc->mapping_table[dev_idx]; for (map_idx = dev_idx; map_idx < (dev_idx + num_slots); map_idx++, mt_entry++) { if (mt_entry->dpm_entry_num != MPR_DPM_BAD_IDX) { - printf("%s: conflict in mapping table " - "for enclosure %d\n", __func__, + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, + "%s: Conflict in mapping table for " + " enclosure %d\n", __func__, enc_idx); break; } - physical_id = dpm_entry->PhysicalIdentifier.High; + physical_id = + dpm_entry->PhysicalIdentifier.High; mt_entry->physical_id = (physical_id << 32) | dpm_entry->PhysicalIdentifier.Low; mt_entry->phy_bits = phy_bits; - mt_entry->channel = 0; mt_entry->id = dev_idx; mt_entry->dpm_entry_num = entry_num; mt_entry->missing_count = missing_cnt; @@ -1954,18 +2354,24 @@ } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + + /* + * Device mapping, so simply copy the DPM entries to the + * mapping table, but check for a corrupt mapping table + * (as described above in Enc/Slot mapping). + */ map_idx = dev_idx; mt_entry = &sc->mapping_table[map_idx]; if (mt_entry->dpm_entry_num != MPR_DPM_BAD_IDX) { - printf("%s: conflict in mapping table for " - "device %d\n", __func__, map_idx); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "Conflict in mapping table for device %d\n", + __func__, map_idx); break; } physical_id = dpm_entry->PhysicalIdentifier.High; mt_entry->physical_id = (physical_id << 32) | dpm_entry->PhysicalIdentifier.Low; mt_entry->phy_bits = phy_bits; - mt_entry->channel = 0; mt_entry->id = dev_idx; mt_entry->missing_count = missing_cnt; mt_entry->dpm_entry_num = entry_num; @@ -1977,43 +2383,91 @@ /* * mpr_mapping_check_devices - start of the day check for device availabilty * @sc: per adapter object - * @sleep_flag: Flag indicating whether this function can sleep or not * * Returns nothing. */ void -mpr_mapping_check_devices(struct mpr_softc *sc, int sleep_flag) +mpr_mapping_check_devices(void *data) { u32 i; -/* u32 cntdn, i; - u32 timeout = 60;*/ struct dev_mapping_table *mt_entry; + struct mpr_softc *sc = (struct mpr_softc *)data; u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags); struct enc_mapping_table *et_entry; - u32 start_idx, end_idx; + u32 start_idx = 0, end_idx = 0; + u8 stop_device_checks = 0; + + MPR_FUNCTRACE(sc); - /* We need to ucomment this when this function is called - * from the port enable complete */ -#if 0 + /* + * Clear this flag so that this function is never called again except + * within this function if the check needs to be done again. The + * purpose is to check for missing devices that are currently in the + * mapping table so do this only at driver init after discovery. + */ sc->track_mapping_events = 0; - cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout; - do { - if (!sc->pending_map_events) - break; - if (sleep_flag == CAN_SLEEP) - pause("mpr_pause", (hz/1000));/* 1msec sleep */ - else - DELAY(500); /* 500 useconds delay */ - } while (--cntdn); + /* + * callout synchronization + * This is used to prevent race conditions for the callout. + */ + mpr_dprint(sc, MPR_MAPPING, "%s: Start check for missing devices.\n", + __func__); + mtx_assert(&sc->mpr_mtx, MA_OWNED); + if ((callout_pending(&sc->device_check_callout)) || + (!callout_active(&sc->device_check_callout))) { + mpr_dprint(sc, MPR_MAPPING, "%s: Device Check Callout is " + "already pending or not active.\n", __func__); + return; + } + callout_deactivate(&sc->device_check_callout); - if (!cntdn) - printf("%s: there are %d" - " pending events after %d seconds of delay\n", - __func__, sc->pending_map_events, timeout); -#endif - sc->pending_map_events = 0; + /* + * Use callout to check if any devices in the mapping table have been + * processed yet. If ALL devices are marked as not init_complete, no + * devices have been processed and mapped. Until devices are mapped + * there's no reason to mark them as missing. Continue resetting this + * callout until devices have been mapped. + */ + if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == + MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { + et_entry = sc->enclosure_table; + for (i = 0; i < sc->num_enc_table_entries; i++, et_entry++) { + if (et_entry->init_complete) { + stop_device_checks = 1; + break; + } + } + } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == + MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + mt_entry = sc->mapping_table; + for (i = 0; i < sc->max_devices; i++, mt_entry++) { + if (mt_entry->init_complete) { + stop_device_checks = 1; + break; + } + } + } + /* + * Setup another callout check after a delay. Keep doing this until + * devices are mapped. + */ + if (!stop_device_checks) { + mpr_dprint(sc, MPR_MAPPING, "%s: No devices have been mapped. " + "Reset callout to check again after a %d second delay.\n", + __func__, MPR_MISSING_CHECK_DELAY); + callout_reset(&sc->device_check_callout, + MPR_MISSING_CHECK_DELAY * hz, mpr_mapping_check_devices, + sc); + return; + } + mpr_dprint(sc, MPR_MAPPING, "%s: Device check complete.\n", __func__); + + /* + * Depending on the mapping type, check if devices have been processed + * and update their missing counts if not processed. + */ if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { et_entry = sc->enclosure_table; @@ -2021,11 +2475,16 @@ if (!et_entry->init_complete) { if (et_entry->missing_count < MPR_MAX_MISSING_COUNT) { + mpr_dprint(sc, MPR_MAPPING, "%s: " + "Enclosure %d is missing from the " + "topology. Update its missing " + "count.\n", __func__, i); et_entry->missing_count++; if (et_entry->dpm_entry_num != - MPR_DPM_BAD_IDX) + MPR_DPM_BAD_IDX) { _mapping_commit_enc_entry(sc, et_entry); + } } et_entry->init_complete = 1; } @@ -2034,69 +2493,45 @@ return; _mapping_get_ir_maprange(sc, &start_idx, &end_idx); mt_entry = &sc->mapping_table[start_idx]; - for (i = start_idx; i < (end_idx + 1); i++, mt_entry++) { - if (mt_entry->device_info & MPR_DEV_RESERVED - && !mt_entry->physical_id) - mt_entry->init_complete = 1; - else if (mt_entry->device_info & MPR_DEV_RESERVED) { - if (!mt_entry->init_complete) { - if (mt_entry->missing_count < - MPR_MAX_MISSING_COUNT) { - mt_entry->missing_count++; - if (mt_entry->dpm_entry_num != - MPR_DPM_BAD_IDX) - _mapping_commit_map_entry(sc, - mt_entry); - } - mt_entry->init_complete = 1; - } - } - } } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + start_idx = 0; + end_idx = sc->max_devices - 1; mt_entry = sc->mapping_table; - for (i = 0; i < sc->max_devices; i++, mt_entry++) { - if (mt_entry->device_info & MPR_DEV_RESERVED - && !mt_entry->physical_id) - mt_entry->init_complete = 1; - else if (mt_entry->device_info & MPR_DEV_RESERVED) { - if (!mt_entry->init_complete) { - if (mt_entry->missing_count < - MPR_MAX_MISSING_COUNT) { - mt_entry->missing_count++; - if (mt_entry->dpm_entry_num != - MPR_DPM_BAD_IDX) + } + + /* + * The start and end indices have been set above according to the + * mapping type. Go through these mappings and update any entries that + * do not have the init_complete flag set, which means they are missing. + */ + if (end_idx == 0) + return; + for (i = start_idx; i < (end_idx + 1); i++, mt_entry++) { + if (mt_entry->device_info & MPR_DEV_RESERVED + && !mt_entry->physical_id) + mt_entry->init_complete = 1; + else if (mt_entry->device_info & MPR_DEV_RESERVED) { + if (!mt_entry->init_complete) { + mpr_dprint(sc, MPR_MAPPING, "%s: Device in " + "mapping table at index %d is missing from " + "topology. Update its missing count.\n", + __func__, i); + if (mt_entry->missing_count < + MPR_MAX_MISSING_COUNT) { + mt_entry->missing_count++; + if (mt_entry->dpm_entry_num != + MPR_DPM_BAD_IDX) { _mapping_commit_map_entry(sc, mt_entry); } - mt_entry->init_complete = 1; } + mt_entry->init_complete = 1; } } } } - -/** - * mpr_mapping_is_reinit_required - check whether event replay required - * @sc: per adapter object - * - * Checks the per ioc flags and decide whether reinit of events required - * - * Returns 1 for reinit of ioc 0 for not. - */ -int mpr_mapping_is_reinit_required(struct mpr_softc *sc) -{ - if (!sc->mt_full_retry && sc->mt_add_device_failed) { - sc->mt_full_retry = 1; - sc->mt_add_device_failed = 0; - _mapping_flush_dpm_pages(sc); - return 1; - } - sc->mt_full_retry = 1; - return 0; -} - /** * mpr_mapping_initialize - initialize mapping tables * @sc: per adapter object @@ -2125,10 +2560,13 @@ sc->pending_map_events = 0; sc->num_enc_table_entries = 0; sc->num_rsvd_entries = 0; - sc->num_channels = 1; sc->max_dpm_entries = sc->ioc_pg8.MaxPersistentEntries; sc->is_dpm_enable = (sc->max_dpm_entries) ? 1 : 0; sc->track_mapping_events = 0; + + mpr_dprint(sc, MPR_MAPPING, "%s: Mapping table has a max of %d entries " + "and DPM has a max of %d entries.\n", __func__, sc->max_devices, + sc->max_dpm_entries); if (ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_DISABLE_PERSISTENT_MAPPING) sc->is_dpm_enable = 0; @@ -2169,8 +2607,8 @@ retry_read_dpm: if (mpr_config_get_dpm_pg0(sc, &mpi_reply, sc->dpm_pg0, dpm_pg0_sz)) { - printf("%s: dpm page read failed; disabling dpm\n", - __func__); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: DPM page " + "read failed.\n", __func__); if (retry_count < 3) { retry_count++; goto retry_read_dpm; @@ -2181,6 +2619,11 @@ if (sc->is_dpm_enable) _mapping_process_dpm_pg0(sc); + else { + mpr_dprint(sc, MPR_MAPPING, "%s: DPM processing is disabled. " + "Device mappings will not persist across reboots or " + "resets.\n", __func__); + } sc->track_mapping_events = 1; return 0; @@ -2200,15 +2643,15 @@ } /** - * mpr_mapping_get_sas_id - assign a target id for sas device + * mpr_mapping_get_tid - return the target id for sas device and handle * @sc: per adapter object * @sas_address: sas address of the device * @handle: device handle * - * Returns valid ID on success or BAD_ID. + * Returns valid target ID on success or BAD_ID. */ unsigned int -mpr_mapping_get_sas_id(struct mpr_softc *sc, uint64_t sas_address, u16 handle) +mpr_mapping_get_tid(struct mpr_softc *sc, uint64_t sas_address, u16 handle) { u32 map_idx; struct dev_mapping_table *mt_entry; @@ -2224,38 +2667,39 @@ } /** - * mpr_mapping_get_sas_id_from_handle - find a target id in mapping table using + * mpr_mapping_get_tid_from_handle - find a target id in mapping table using * only the dev handle. This is just a wrapper function for the local function * _mapping_get_mt_idx_from_handle. * @sc: per adapter object * @handle: device handle * - * Returns valid ID on success or BAD_ID. + * Returns valid target ID on success or BAD_ID. */ unsigned int -mpr_mapping_get_sas_id_from_handle(struct mpr_softc *sc, u16 handle) +mpr_mapping_get_tid_from_handle(struct mpr_softc *sc, u16 handle) { return (_mapping_get_mt_idx_from_handle(sc, handle)); } /** - * mpr_mapping_get_raid_id - assign a target id for raid device + * mpr_mapping_get_raid_tid - return the target id for raid device * @sc: per adapter object * @wwid: world wide identifier for raid volume - * @handle: device handle + * @volHandle: volume device handle * - * Returns valid ID on success or BAD_ID. + * Returns valid target ID on success or BAD_ID. */ unsigned int -mpr_mapping_get_raid_id(struct mpr_softc *sc, u64 wwid, u16 handle) +mpr_mapping_get_raid_tid(struct mpr_softc *sc, u64 wwid, u16 volHandle) { - u32 map_idx; + u32 start_idx, end_idx, map_idx; struct dev_mapping_table *mt_entry; - for (map_idx = 0; map_idx < sc->max_devices; map_idx++) { - mt_entry = &sc->mapping_table[map_idx]; - if (mt_entry->dev_handle == handle && mt_entry->physical_id == - wwid) + _mapping_get_ir_maprange(sc, &start_idx, &end_idx); + mt_entry = &sc->mapping_table[start_idx]; + for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) { + if (mt_entry->dev_handle == volHandle && + mt_entry->physical_id == wwid) return mt_entry->id; } @@ -2263,16 +2707,16 @@ } /** - * mpr_mapping_get_raid_id_from_handle - find raid device in mapping table + * mpr_mapping_get_raid_tid_from_handle - find raid device in mapping table * using only the volume dev handle. This is just a wrapper function for the * local function _mapping_get_ir_mt_idx_from_handle. * @sc: per adapter object * @volHandle: volume device handle * - * Returns valid ID on success or BAD_ID. + * Returns valid target ID on success or BAD_ID. */ unsigned int -mpr_mapping_get_raid_id_from_handle(struct mpr_softc *sc, u16 volHandle) +mpr_mapping_get_raid_tid_from_handle(struct mpr_softc *sc, u16 volHandle) { return (_mapping_get_ir_mt_idx_from_handle(sc, volHandle)); } @@ -2306,8 +2750,8 @@ if (event_data->ReasonCode == MPI2_EVENT_SAS_ENCL_RC_ADDED) { if (!event_data->NumSlots) { - printf("%s: enclosure with handle = 0x%x reported 0 " - "slots\n", __func__, + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: Enclosure " + "with handle = 0x%x reported 0 slots.\n", __func__, le16toh(event_data->EnclosureHandle)); goto out; } @@ -2316,13 +2760,22 @@ event_data->EnclosureLogicalID.Low; enc_idx = _mapping_get_enc_idx_from_id(sc, temp64_var, event_data->PhyBits); + + /* + * If the Added enclosure is already in the Enclosure Table, + * make sure that all the the enclosure info is up to date. If + * the enclosure was missing and has just been added back, or if + * the enclosure's Phy Bits have changed, clear the missing + * count and update the Phy Bits in the mapping table and in the + * DPM, if it's being used. + */ if (enc_idx != MPR_ENCTABLE_BAD_IDX) { et_entry = &sc->enclosure_table[enc_idx]; if (et_entry->init_complete && !et_entry->missing_count) { - printf("%s: enclosure %d is already present " - "with handle = 0x%x\n",__func__, enc_idx, - et_entry->enc_handle); + mpr_dprint(sc, MPR_MAPPING, "%s: Enclosure %d " + "is already present with handle = 0x%x\n", + __func__, enc_idx, et_entry->enc_handle); goto out; } et_entry->enc_handle = le16toh(event_data-> @@ -2341,8 +2794,7 @@ missing_count = (u8)(dpm_entry->MappingInformation & MPI2_DRVMAP0_MAPINFO_MISSING_MASK); - if (!et_entry->init_complete && ( - missing_count || update_phy_bits)) { + if (missing_count || update_phy_bits) { dpm_entry->MappingInformation = et_entry->num_slots; dpm_entry->MappingInformation @@ -2355,20 +2807,29 @@ } } } else { + /* + * This is a new enclosure that is being added. + * Initialize the Enclosure Table entry. It will be + * finalized when a device is added for the enclosure + * and the enclosure has enough space in the Mapping + * Table to map its devices. + */ enc_idx = sc->num_enc_table_entries; if (enc_idx >= sc->max_enclosures) { - printf("%s: enclosure can not be added; " - "mapping table is full\n", __func__); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: " + "Enclosure cannot be added to mapping " + "table because it's full.\n", __func__); goto out; } sc->num_enc_table_entries++; et_entry = &sc->enclosure_table[enc_idx]; et_entry->enc_handle = le16toh(event_data-> EnclosureHandle); - et_entry->enclosure_id = event_data-> - EnclosureLogicalID.High; - et_entry->enclosure_id = ( et_entry->enclosure_id << - 32) | event_data->EnclosureLogicalID.Low; + et_entry->enclosure_id = le64toh(event_data-> + EnclosureLogicalID.High); + et_entry->enclosure_id = + ((et_entry->enclosure_id << 32) | + le64toh(event_data->EnclosureLogicalID.Low)); et_entry->start_index = MPR_MAPTABLE_BAD_IDX; et_entry->dpm_entry_num = MPR_DPM_BAD_IDX; et_entry->num_slots = le16toh(event_data->NumSlots); @@ -2378,23 +2839,23 @@ et_entry->init_complete = 1; } else if (event_data->ReasonCode == MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING) { + /* + * An enclosure was removed. Update its missing count and then + * update the DPM entry with the new missing count for the + * enclosure. + */ enc_idx = _mapping_get_enc_idx_from_handle(sc, le16toh(event_data->EnclosureHandle)); if (enc_idx == MPR_ENCTABLE_BAD_IDX) { - printf("%s: cannot unmap enclosure %d because it has " - "already been deleted", __func__, enc_idx); + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, "%s: Cannot " + "unmap enclosure %d because it has already been " + "deleted.\n", __func__, enc_idx); goto out; } et_entry = &sc->enclosure_table[enc_idx]; - if (!et_entry->init_complete) { - if (et_entry->missing_count < MPR_MAX_MISSING_COUNT) - et_entry->missing_count++; - else - et_entry->init_complete = 1; - } - if (!et_entry->missing_count) + if (et_entry->missing_count < MPR_MAX_MISSING_COUNT) et_entry->missing_count++; - if (sc->is_dpm_enable && !et_entry->init_complete && + if (sc->is_dpm_enable && et_entry->dpm_entry_num != MPR_DPM_BAD_IDX) { dpm_entry += et_entry->dpm_entry_num; dpm_entry->MappingInformation = et_entry->num_slots; @@ -2514,66 +2975,6 @@ } /** - * _mapping_check_update_ir_mt_idx - Check and update IR map table index - * @sc: per adapter object - * @event_data: event data payload - * @evt_idx: current event index - * @map_idx: current index and the place holder for new map table index - * @wwid_table: world wide name for volumes in the element table - * - * pass through IR events and find whether any events matches and if so - * tries to find new index if not returns failure - * - * Returns 0 on success and 1 on failure - */ -static int -_mapping_check_update_ir_mt_idx(struct mpr_softc *sc, - Mpi2EventDataIrConfigChangeList_t *event_data, int evt_idx, u32 *map_idx, - u64 *wwid_table) -{ - struct dev_mapping_table *mt_entry; - u32 st_idx, end_idx, mt_idx = *map_idx; - u8 match = 0; - Mpi2EventIrConfigElement_t *element; - u16 element_flags; - int i; - - mt_entry = &sc->mapping_table[mt_idx]; - _mapping_get_ir_maprange(sc, &st_idx, &end_idx); -search_again: - match = 0; - for (i = evt_idx + 1; i < event_data->NumElements; i++) { - element = (Mpi2EventIrConfigElement_t *) - &event_data->ConfigElement[i]; - element_flags = le16toh(element->ElementFlags); - if ((element_flags & - MPI2_EVENT_IR_CHANGE_EFLAGS_ELEMENT_TYPE_MASK) != - MPI2_EVENT_IR_CHANGE_EFLAGS_VOLUME_ELEMENT) - continue; - if (element->ReasonCode == MPI2_EVENT_IR_CHANGE_RC_ADDED || - element->ReasonCode == - MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED) { - if (mt_entry->physical_id == wwid_table[i]) { - match = 1; - break; - } - } - } - - if (match) { - do { - mt_idx++; - if (mt_idx > end_idx) - return 1; - mt_entry = &sc->mapping_table[mt_idx]; - } while (mt_entry->device_info & MPR_MAP_IN_USE); - goto search_again; - } - *map_idx = mt_idx; - return 0; -} - -/** * mpr_mapping_ir_config_change_event - handle IR config change list events * @sc: per adapter object * @event_data: event data payload @@ -2590,7 +2991,6 @@ u32 map_idx, flags; struct dev_mapping_table *mt_entry; u16 element_flags; - u8 log_full_error = 0; wwid_table = malloc(sizeof(u64) * event_data->NumElements, M_MPR, M_NOWAIT | M_ZERO); @@ -2598,6 +2998,11 @@ goto out; element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0]; flags = le32toh(event_data->Flags); + + /* + * For volume changes, get the WWID for the volume and put it in a + * table to be used in the processing of the IR change event. + */ for (i = 0; i < event_data->NumElements; i++, element++) { element_flags = le16toh(element->ElementFlags); if ((element->ReasonCode != MPI2_EVENT_IR_CHANGE_RC_ADDED) && @@ -2611,14 +3016,14 @@ MPI2_EVENT_IR_CHANGE_EFLAGS_VOLUME_ELEMENT) { mpr_config_get_volume_wwid(sc, le16toh(element->VolDevHandle), &wwid_table[i]); - map_idx = _mapping_get_ir_mt_idx_from_wwid(sc, - wwid_table[i]); - if (map_idx != MPR_MAPTABLE_BAD_IDX) { - mt_entry = &sc->mapping_table[map_idx]; - mt_entry->device_info |= MPR_MAP_IN_USE; - } } } + + /* + * Check the ReasonCode for each element in the IR event and Add/Remove + * Volumes or Physical Disks of Volumes to/from the mapping table. Use + * the WWIDs gotten above in wwid_table. + */ if (flags == MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG) goto out; else { @@ -2632,8 +3037,11 @@ map_idx = _mapping_get_ir_mt_idx_from_wwid (sc, wwid_table[i]); if (map_idx != MPR_MAPTABLE_BAD_IDX) { + /* + * The volume is already in the mapping + * table. Just update it's info. + */ mt_entry = &sc->mapping_table[map_idx]; - mt_entry->channel = MPR_RAID_CHANNEL; mt_entry->id = map_idx; mt_entry->dev_handle = le16toh (element->VolDevHandle); @@ -2643,31 +3051,33 @@ map_idx, element, wwid_table[i]); continue; } + + /* + * Volume is not in mapping table yet. Find a + * free entry in the mapping table at the + * volume mapping locations. If no entries are + * available, this is an error because it means + * there are more volumes than can be mapped + * and that should never happen for volumes. + */ map_idx = _mapping_get_free_ir_mt_idx(sc); if (map_idx == MPR_MAPTABLE_BAD_IDX) - log_full_error = 1; - else if (i < (event_data->NumElements - 1)) { - log_full_error = - _mapping_check_update_ir_mt_idx - (sc, event_data, i, &map_idx, - wwid_table); - } - if (log_full_error) { - printf("%s: no space to add the RAID " - "volume with handle 0x%04x in " - "mapping table\n", __func__, le16toh - (element->VolDevHandle)); + { + mpr_dprint(sc, MPR_ERROR | MPR_MAPPING, + "%s: failed to add the volume with " + "handle 0x%04x because there is no " + "free space available in the " + "mapping table\n", __func__, + le16toh(element->VolDevHandle)); continue; } mt_entry = &sc->mapping_table[map_idx]; mt_entry->physical_id = wwid_table[i]; - mt_entry->channel = MPR_RAID_CHANNEL; mt_entry->id = map_idx; mt_entry->dev_handle = le16toh(element-> VolDevHandle); mt_entry->device_info = MPR_DEV_RESERVED | MPR_MAP_IN_USE; - mt_entry->init_complete = 0; _mapping_update_ir_missing_cnt(sc, map_idx, element, wwid_table[i]); } else if (element->ReasonCode == @@ -2675,9 +3085,10 @@ map_idx = _mapping_get_ir_mt_idx_from_wwid(sc, wwid_table[i]); if (map_idx == MPR_MAPTABLE_BAD_IDX) { - printf("%s: failed to remove a volume " - "because it has already been " - "removed\n", __func__); + mpr_dprint(sc, MPR_MAPPING,"%s: Failed " + "to remove a volume because it has " + "already been removed.\n", + __func__); continue; } _mapping_update_ir_missing_cnt(sc, map_idx, @@ -2687,9 +3098,10 @@ map_idx = _mapping_get_mt_idx_from_handle(sc, le16toh(element->VolDevHandle)); if (map_idx == MPR_MAPTABLE_BAD_IDX) { - printf("%s: failed to remove volume " - "with handle 0x%04x because it has " - "already been removed\n", __func__, + mpr_dprint(sc, MPR_MAPPING,"%s: Failed " + "to remove volume with handle " + "0x%04x because it has already " + "been removed.\n", __func__, le16toh(element->VolDevHandle)); continue; } Index: head/sys/dev/mpr/mpr_sas.c =================================================================== --- head/sys/dev/mpr/mpr_sas.c +++ head/sys/dev/mpr/mpr_sas.c @@ -745,7 +745,7 @@ * of MaxTargets here so that we don't get into trouble later. This * should move into the reinit logic. */ - sassc->maxtargets = sc->facts->MaxTargets; + sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes; sassc->targets = malloc(sizeof(struct mprsas_target) * sassc->maxtargets, M_MPR, M_WAITOK|M_ZERO); if (!sassc->targets) { @@ -963,6 +963,25 @@ if (sassc->flags & MPRSAS_DISCOVERY_TIMEOUT_PENDING) callout_stop(&sassc->discovery_callout); + /* + * After discovery has completed, check the mapping table for any + * missing devices and update their missing counts. Only do this once + * whenever the driver is initialized so that missing counts aren't + * updated unnecessarily. Note that just because discovery has + * completed doesn't mean that events have been processed yet. The + * check_devices function is a callout timer that checks if ALL devices + * are missing. If so, it will wait a little longer for events to + * complete and keep resetting itself until some device in the mapping + * table is not missing, meaning that event processing has started. + */ + if (sc->track_mapping_events) { + mpr_dprint(sc, MPR_XINFO | MPR_MAPPING, "Discovery has " + "completed. Check for missing devices in the mapping " + "table.\n"); + callout_reset(&sc->device_check_callout, + MPR_MISSING_CHECK_DELAY * hz, mpr_mapping_check_devices, + sc); + } } static void @@ -996,7 +1015,12 @@ cpi->hba_eng_cnt = 0; cpi->max_target = sassc->maxtargets - 1; cpi->max_lun = 255; - cpi->initiator_id = sassc->maxtargets - 1; + + /* + * initiator_id is set here to an ID outside the set of valid + * target IDs (including volumes). + */ + cpi->initiator_id = sassc->maxtargets; strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); Index: head/sys/dev/mpr/mpr_sas_lsi.c =================================================================== --- head/sys/dev/mpr/mpr_sas_lsi.c +++ head/sys/dev/mpr/mpr_sas_lsi.c @@ -223,8 +223,9 @@ if (mprsas_add_device(sc, le16toh(phy->AttachedDevHandle), phy->LinkRate)) { - printf("%s: failed to add device with " - "handle 0x%x\n", __func__, + mpr_dprint(sc, MPR_ERROR, "%s: " + "failed to add device with handle " + "0x%x\n", __func__, le16toh(phy->AttachedDevHandle)); mprsas_prepare_remove(sassc, le16toh( phy->AttachedDevHandle)); @@ -289,7 +290,7 @@ element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0]; - id = mpr_mapping_get_raid_id_from_handle(sc, + id = mpr_mapping_get_raid_tid_from_handle(sc, element->VolDevHandle); mpr_mapping_ir_config_change_event(sc, event_data); @@ -833,10 +834,17 @@ * 1 - use the PhyNum field as a fallback to the mapping logic * 0 - never use the PhyNum field * -1 - only use the PhyNum field + * + * Note that using the Phy number to map a device can cause device adds + * to fail if multiple enclosures/expanders are in the topology. For + * example, if two devices are in the same slot number in two different + * enclosures within the topology, only one of those devices will be + * added. PhyNum mapping should not be used if multiple enclosures are + * in the topology. */ id = MPR_MAP_BAD_ID; if (sc->use_phynum != -1) - id = mpr_mapping_get_sas_id(sc, sas_address, handle); + id = mpr_mapping_get_tid(sc, sas_address, handle); if (id == MPR_MAP_BAD_ID) { if ((sc->use_phynum == 0) || ((id = config_page.PhyNum) > sassc->maxtargets)) { @@ -847,19 +855,31 @@ goto out; } } + mpr_dprint(sc, MPR_MAPPING, "%s: Target ID for added device is %d.\n", + __func__, id); - if (mprsas_check_id(sassc, id) != 0) { - device_printf(sc->mpr_dev, "Excluding target id %d\n", id); - error = ENXIO; - goto out; - } - + /* + * Only do the ID check and reuse check if the target is not from a + * RAID Component. For Physical Disks of a Volume, the ID will be reused + * when a volume is deleted because the mapping entry for the PD will + * still be in the mapping table. The ID check should not be done here + * either since this PD is already being used. + */ targ = &sassc->targets[id]; - if (targ->handle != 0x0) { - mpr_dprint(sc, MPR_MAPPING, "Attempting to reuse target id " - "%d handle 0x%04x\n", id, targ->handle); - error = ENXIO; - goto out; + if (!(targ->flags & MPR_TARGET_FLAGS_RAID_COMPONENT)) { + if (mprsas_check_id(sassc, id) != 0) { + device_printf(sc->mpr_dev, "Excluding target id %d\n", + id); + error = ENXIO; + goto out; + } + + if (targ->handle != 0x0) { + mpr_dprint(sc, MPR_MAPPING, "Attempting to reuse " + "target id %d handle 0x%04x\n", id, targ->handle); + error = ENXIO; + goto out; + } } mpr_dprint(sc, MPR_MAPPING, "SAS Address from SAS device page0 = %jx\n", @@ -1256,14 +1276,16 @@ goto out; } - id = mpr_mapping_get_sas_id(sc, pcie_wwid, handle); + id = mpr_mapping_get_tid(sc, pcie_wwid, handle); if (id == MPR_MAP_BAD_ID) { - printf("failure at %s:%d/%s()! Could not get ID for device " - "with handle 0x%04x\n", __FILE__, __LINE__, __func__, - handle); + mpr_dprint(sc, MPR_ERROR | MPR_INFO, "failure at %s:%d/%s()! " + "Could not get ID for device with handle 0x%04x\n", + __FILE__, __LINE__, __func__, handle); error = ENXIO; goto out; } + mpr_dprint(sc, MPR_MAPPING, "%s: Target ID for added device is %d.\n", + __func__, id); if (mprsas_check_id(sassc, id) != 0) { device_printf(sc->mpr_dev, "Excluding target id %d\n", id); @@ -1356,7 +1378,7 @@ goto out; } - id = mpr_mapping_get_raid_id(sc, wwid, handle); + id = mpr_mapping_get_raid_tid(sc, wwid, handle); if (id == MPR_MAP_BAD_ID) { printf("%s: could not get ID for volume with handle 0x%04x and " "WWID 0x%016llx\n", __func__, handle, @@ -1418,7 +1440,7 @@ */ sc->SSU_started = TRUE; sc->SSU_refcount = 0; - for (targetid = 0; targetid < sc->facts->MaxTargets; targetid++) { + for (targetid = 0; targetid < sc->max_devices; targetid++) { target = &sassc->targets[targetid]; if (target->handle == 0x0) { continue; @@ -1602,7 +1624,7 @@ * 3: enable to SSD and HDD * anything else will default to 1. */ - for (targetid = 0; targetid < sc->facts->MaxTargets; targetid++) { + for (targetid = 0; targetid < sc->max_devices; targetid++) { target = &sc->sassc->targets[targetid]; if (target->handle == 0x0) { continue; Index: head/sys/dev/mpr/mpr_user.c =================================================================== --- head/sys/dev/mpr/mpr_user.c +++ head/sys/dev/mpr/mpr_user.c @@ -2145,7 +2145,7 @@ data->DevHandle = dev_handle; } else { bus = 0; - target = mpr_mapping_get_sas_id_from_handle(sc, dev_handle); + target = mpr_mapping_get_tid_from_handle(sc, dev_handle); data->Bus = bus; data->TargetID = target; } Index: head/sys/dev/mpr/mprvar.h =================================================================== --- head/sys/dev/mpr/mprvar.h +++ head/sys/dev/mpr/mprvar.h @@ -33,7 +33,7 @@ #ifndef _MPRVAR_H #define _MPRVAR_H -#define MPR_DRIVER_VERSION "15.02.00.00-fbsd" +#define MPR_DRIVER_VERSION "15.03.00.00-fbsd" #define MPR_DB_MAX_WAIT 2500 @@ -69,6 +69,7 @@ #define MPR_PERIODIC_DELAY 1 /* 1 second heartbeat/watchdog check */ #define MPR_ATA_ID_TIMEOUT 5 /* 5 second timeout for SATA ID cmd */ +#define MPR_MISSING_CHECK_DELAY 10 /* 10 seconds between missing check */ #define IFAULT_IOP_OVER_TEMP_THRESHOLD_EXCEEDED 0x2810 @@ -88,7 +89,6 @@ #define MPR_MAX_MISSING_COUNT 0x0F #define MPR_DEV_RESERVED 0x20000000 #define MPR_MAP_IN_USE 0x10000000 -#define MPR_RAID_CHANNEL 1 #define MPR_MAP_BAD_ID 0xFFFFFFFF typedef uint8_t u8; @@ -103,7 +103,6 @@ * @phy_bits: bitfields indicating controller phys * @dpm_entry_num: index of this device in device persistent map table * @dev_handle: device handle for the device pointed by this entry - * @channel: target channel * @id: target id * @missing_count: number of times the device not detected by driver * @hide_flag: Hide this physical disk/not (foreign configuration) @@ -116,8 +115,7 @@ u32 phy_bits; u16 dpm_entry_num; u16 dev_handle; - u8 reserved1; - u8 channel; + u16 reserved1; u16 id; u8 missing_count; u8 init_complete; @@ -306,6 +304,7 @@ struct mpr_chain *chains; struct mpr_prp_page *prps; struct callout periodic; + struct callout device_check_callout; struct mprsas_softc *sassc; char tmp_string[MPR_STRING_LENGTH]; @@ -397,13 +396,10 @@ uint8_t max_volumes; uint8_t num_enc_table_entries; uint8_t num_rsvd_entries; - uint8_t num_channels; uint16_t max_dpm_entries; uint8_t is_dpm_enable; uint8_t track_mapping_events; uint32_t pending_map_events; - uint8_t mt_full_retry; - uint8_t mt_add_device_failed; /* FW diag Buffer List */ mpr_fw_diagnostic_buffer_t @@ -774,19 +770,18 @@ Mpi2EventDataSasTopologyChangeList_t *); void mpr_mapping_pcie_topology_change_event(struct mpr_softc *sc, Mpi26EventDataPCIeTopologyChangeList_t *event_data); -int mpr_mapping_is_reinit_required(struct mpr_softc *); void mpr_mapping_free_memory(struct mpr_softc *sc); int mpr_config_set_dpm_pg0(struct mpr_softc *, Mpi2ConfigReply_t *, Mpi2DriverMappingPage0_t *, u16 ); void mpr_mapping_exit(struct mpr_softc *); -void mpr_mapping_check_devices(struct mpr_softc *, int); +void mpr_mapping_check_devices(void *); int mpr_mapping_allocate_memory(struct mpr_softc *sc); -unsigned int mpr_mapping_get_sas_id(struct mpr_softc *, uint64_t , u16); -unsigned int mpr_mapping_get_sas_id_from_handle(struct mpr_softc *sc, +unsigned int mpr_mapping_get_tid(struct mpr_softc *, uint64_t , u16); +unsigned int mpr_mapping_get_tid_from_handle(struct mpr_softc *sc, u16 handle); -unsigned int mpr_mapping_get_raid_id(struct mpr_softc *sc, u64 wwid, - u16 handle); -unsigned int mpr_mapping_get_raid_id_from_handle(struct mpr_softc *sc, +unsigned int mpr_mapping_get_raid_tid(struct mpr_softc *sc, u64 wwid, + u16 volHandle); +unsigned int mpr_mapping_get_raid_tid_from_handle(struct mpr_softc *sc, u16 volHandle); void mpr_mapping_enclosure_dev_status_change_event(struct mpr_softc *, Mpi2EventDataSasEnclDevStatusChange_t *event_data);