Index: share/man/man4/mps.4 =================================================================== --- share/man/man4/mps.4 +++ share/man/man4/mps.4 @@ -1,8 +1,8 @@ .\" .\" Copyright (c) 2010 Spectra Logic Corporation .\" Copyright (c) 2014 LSI Corp -.\" Copyright (c) 2016 Avago Technologies -.\" Copyright (c) 2016 Broadcom Ltd. +.\" Copyright (c) 2017 Avago Technologies +.\" Copyright (c) 2017 Broadcom Ltd. .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -38,7 +38,7 @@ .\" $Id: //depot/SpectraBSD/head/share/man/man4/mps.4#6 $ .\" $FreeBSD$ .\" -.Dd July 5, 2016 +.Dd May 25, 2017 .Dt MPS 4 .Os .Sh NAME @@ -241,13 +241,13 @@ Send SSU to both HDDs and SSDs. .El .Pp -To control the feature for a specific adapter, set this tunable value in +To control this feature for a specific adapter, set this tunable value in .Xr loader.conf 5 : .Bd -literal -offset indent dev.mps.X.enable_ssu .Ed .Pp -The same set of values are valid when setting this tunable for all adapters. +The same set of values are valid as when setting this tunable for all adapters. .Pp SATA disks that take several seconds to spin up and fail the SATA Identify command might not be discovered by the driver. @@ -273,6 +273,45 @@ tunable. NNNN is the number of seconds to wait for SATA devices to spin up when they fail the initial SATA Identify command. +.Pp +The driver can map devices discovered by the adapter so that target IDs +corresponding to a specific device persist across resets and reboots. +In some cases it is possible for devices to lose their mapped IDs due to +unexpected behavior from certain hardware, such as some types of enclosures. +To overcome this problem, a tunable is provided that will force the driver to +map devices using the Phy number associated with the device. +This feature is not recommended if the topology includes multiple +enclosures/expanders. +If multiple enclosures/expanders are present in the topology, Phy numbers are +repeated, causing all devices at these Phy numbers except the first device to +fail enumeration. +To control this feature for all adapters, set the +.Bd -literal -offset indent +hw.mps.use_phy_num +.Ed +.Pp +tunable in +.Xr loader.conf 5 +to one of these values: +.Bl -tag -width 6n -offset indent +.It -1 +Only use Phy numbers to map devices and bypass the driver's mapping logic. +.It 0 +Never use Phy numbers to map devices. +.It 1 +Use Phy numbers to map devices, but only if the driver's mapping logic fails +to map the device that is being enumerated. +This is the default value. +.El +.Pp +To control this feature for a specific adapter, set this tunable value in +.Xr loader.conf 5 : +.Bd -literal -offset indent +dev.mps.X.use_phy_num +.Ed +.Pp +The same set of values are valid as when setting this tunable for all adapters. +.Pp .Sh DEBUGGING To enable debugging prints from the .Nm Index: sys/dev/mps/mps.c =================================================================== --- sys/dev/mps/mps.c +++ sys/dev/mps/mps.c @@ -505,7 +505,8 @@ */ if (reallocating) { mps_iocfacts_free(sc); - mpssas_realloc_targets(sc, saved_facts.MaxTargets); + mpssas_realloc_targets(sc, saved_facts.MaxTargets + + saved_facts.MaxVolumes); } /* @@ -1518,6 +1519,7 @@ mtx_init(&sc->mps_mtx, "MPT2SAS lock", NULL, MTX_DEF); callout_init_mtx(&sc->periodic, &sc->mps_mtx, 0); + callout_init_mtx(&sc->device_check_callout, &sc->mps_mtx, 0); TAILQ_INIT(&sc->event_list); timevalclear(&sc->lastfail); @@ -1682,6 +1684,7 @@ mps_unlock(sc); /* Lock must not be held for this */ callout_drain(&sc->periodic); + callout_drain(&sc->device_check_callout); if (((error = mps_detach_log(sc)) != 0) || ((error = mps_detach_sas(sc)) != 0)) Index: sys/dev/mps/mps_mapping.c =================================================================== --- sys/dev/mps/mps_mapping.c +++ sys/dev/mps/mps_mapping.c @@ -60,7 +60,7 @@ #include /** - * _mapping_clear_entry - Clear a particular mapping entry. + * _mapping_clear_map_entry - Clear a particular mapping entry. * @map_entry: map table entry * * Returns nothing. @@ -73,7 +73,6 @@ map_entry->phy_bits = 0; map_entry->dpm_entry_num = MPS_DPM_BAD_IDX; map_entry->dev_handle = 0; - map_entry->channel = -1; map_entry->id = -1; map_entry->missing_count = 0; map_entry->init_complete = 0; @@ -140,12 +139,15 @@ dpm_entry->PhysicalBitsMapping = htole32(et_entry->phy_bits); dpm_entry->Reserved1 = 0; + mps_dprint(sc, MPS_MAPPING, "%s: Writing DPM entry %d for enclosure.\n", + __func__, et_entry->dpm_entry_num); memcpy(&config_page.Entry, (u8 *)dpm_entry, sizeof(Mpi2DriverMap0Entry_t)); if (mps_config_set_dpm_pg0(sc, &mpi_reply, &config_page, et_entry->dpm_entry_num)) { - printf("%s: write of dpm entry %d for enclosure failed\n", - __func__, et_entry->dpm_entry_num); + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: Write of DPM " + "entry %d for enclosure failed.\n", __func__, + et_entry->dpm_entry_num); dpm_entry->MappingInformation = le16toh(dpm_entry-> MappingInformation); dpm_entry->DeviceIndex = le16toh(dpm_entry->DeviceIndex); @@ -164,7 +166,7 @@ /** * _mapping_commit_map_entry - write a particular map table entry in DPM page0. * @sc: per adapter object - * @enc_entry: enclosure table entry + * @mt_entry: mapping table entry * * Returns 0 for success, non-zero for failure. */ @@ -180,6 +182,19 @@ if (!sc->is_dpm_enable) return 0; + /* + * It's possible that this Map Entry points to a BAD DPM index. This + * can happen if the Map Entry is a for a missing device and the DPM + * entry that was being used by this device is now being used by some + * new device. So, check for a BAD DPM index and just return if so. + */ + if (mt_entry->dpm_entry_num == MPS_DPM_BAD_IDX) { + mps_dprint(sc, MPS_MAPPING, "%s: DPM entry location for target " + "%d is invalid. DPM will not be written.\n", __func__, + mt_entry->id); + return 0; + } + memset(&config_page, 0, sizeof(Mpi2DriverMappingPage0_t)); memcpy(&config_page.Header, (u8 *)sc->dpm_pg0, sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); @@ -193,13 +208,16 @@ dpm_entry->MappingInformation = htole16(mt_entry->missing_count); dpm_entry->PhysicalBitsMapping = 0; dpm_entry->Reserved1 = 0; - dpm_entry->MappingInformation = htole16(dpm_entry->MappingInformation); memcpy(&config_page.Entry, (u8 *)dpm_entry, sizeof(Mpi2DriverMap0Entry_t)); + + mps_dprint(sc, MPS_MAPPING, "%s: Writing DPM entry %d for target %d.\n", + __func__, mt_entry->dpm_entry_num, mt_entry->id); if (mps_config_set_dpm_pg0(sc, &mpi_reply, &config_page, mt_entry->dpm_entry_num)) { - printf("%s: write of dpm entry %d for device failed\n", - __func__, mt_entry->dpm_entry_num); + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: Write of DPM " + "entry %d for target %d failed.\n", __func__, + mt_entry->dpm_entry_num, mt_entry->id); dpm_entry->MappingInformation = le16toh(dpm_entry-> MappingInformation); dpm_entry->DeviceIndex = le16toh(dpm_entry->DeviceIndex); @@ -307,7 +325,7 @@ et_entry = &sc->enclosure_table[enc_idx]; if ((et_entry->missing_count > high_missing_count) && !et_entry->skip_search) { - high_missing_count = et_entry->missing_count; + high_missing_count = et_entry->missing_count; high_idx = enc_idx; } } @@ -326,7 +344,7 @@ static u32 _mapping_get_high_missing_mt_idx(struct mps_softc *sc) { - u32 map_idx, high_idx = MPS_ENCTABLE_BAD_IDX; + u32 map_idx, high_idx = MPS_MAPTABLE_BAD_IDX; u8 high_missing_count = 0; u32 start_idx, end_idx, start_idx_ir, end_idx_ir; struct dev_mapping_table *mt_entry; @@ -370,7 +388,7 @@ _mapping_get_ir_maprange(sc, &start_idx, &end_idx); mt_entry = &sc->mapping_table[start_idx]; - for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) + for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) if (mt_entry->physical_id == wwid) return map_idx; @@ -458,20 +476,32 @@ u32 high_idx = MPS_MAPTABLE_BAD_IDX; struct dev_mapping_table *mt_entry; + /* + * The IN_USE flag should be clear if the entry is available to use. + * This flag is cleared on initialization and and when a volume is + * deleted. All other times this flag should be set. If, for some + * reason, a free entry cannot be found, look for the entry with the + * highest missing count just in case there is one. + */ _mapping_get_ir_maprange(sc, &start_idx, &end_idx); mt_entry = &sc->mapping_table[start_idx]; - for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) + for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) { if (!(mt_entry->device_info & MPS_MAP_IN_USE)) return map_idx; - mt_entry = &sc->mapping_table[start_idx]; - for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) { if (mt_entry->missing_count > high_missing_count) { high_missing_count = mt_entry->missing_count; high_idx = map_idx; } } + + if (high_idx == MPS_MAPTABLE_BAD_IDX) { + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: Could not find a " + "free entry in the mapping table for a Volume. The mapping " + "table is probably corrupt.\n", __func__); + } + return high_idx; } @@ -494,6 +524,7 @@ if (sc->ir_firmware && (volume_mapping_flags == MPI2_IOCPAGE8_IRFLAGS_HIGH_VOLUME_MAPPING)) max_idx -= sc->max_volumes; + for (map_idx = start_idx; map_idx < max_idx; map_idx++, mt_entry++) if (!(mt_entry->device_info & (MPS_MAP_IN_USE | MPS_DEV_RESERVED))) @@ -542,12 +573,66 @@ _mapping_get_free_dpm_idx(struct mps_softc *sc) { u16 entry_num; + Mpi2DriverMap0Entry_t *dpm_entry; + u16 current_entry = MPS_DPM_BAD_IDX, missing_cnt, high_missing_cnt = 0; + u64 physical_id; + struct dev_mapping_table *mt_entry; + u32 map_idx; - for (entry_num = 0; entry_num < sc->max_dpm_entries; entry_num++) { - if (!sc->dpm_entry_used[entry_num]) - return entry_num; + for (entry_num = 0; entry_num < sc->max_dpm_entries; entry_num++) { + dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); + dpm_entry += entry_num; + missing_cnt = dpm_entry->MappingInformation & + MPI2_DRVMAP0_MAPINFO_MISSING_MASK; + + /* + * If entry is used and not missing, then this entry can't be + * used. Look at next one. + */ + if (sc->dpm_entry_used[entry_num] && !missing_cnt) + continue; + + /* + * If this entry is not used at all, then the missing count + * doesn't matter. Just use this one. Otherwise, keep looking + * and make sure the entry with the highest missing count is + * used. + */ + if (!sc->dpm_entry_used[entry_num]) { + current_entry = entry_num; + break; + } + if ((current_entry == MPS_DPM_BAD_IDX) || + (missing_cnt > high_missing_cnt)) { + current_entry = entry_num; + high_missing_cnt = missing_cnt; + } + } + + /* + * If an entry has been found to use and it's already marked as used + * it means that some device was already using this entry but it's + * missing, and that means that the connection between the missing + * device's DPM entry and the mapping table needs to be cleared. To do + * this, use the Physical ID of the old device still in the DPM entry + * to find its mapping table entry, then mark its DPM entry as BAD. + */ + if ((current_entry != MPS_DPM_BAD_IDX) && + sc->dpm_entry_used[current_entry]) { + dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); + dpm_entry += current_entry; + physical_id = dpm_entry->PhysicalIdentifier.High; + physical_id = (physical_id << 32) | + dpm_entry->PhysicalIdentifier.Low; + map_idx = _mapping_get_mt_idx_from_id(sc, physical_id); + if (map_idx != MPS_MAPTABLE_BAD_IDX) { + mt_entry = &sc->mapping_table[map_idx]; + mt_entry->dpm_entry_num = MPS_DPM_BAD_IDX; + } } - return MPS_DPM_BAD_IDX; + return current_entry; } /** @@ -566,40 +651,57 @@ Mpi2EventIrConfigElement_t *element, u64 wwid) { struct dev_mapping_table *mt_entry; - u8 missing_cnt, reason = element->ReasonCode; + u8 missing_cnt, reason = element->ReasonCode, update_dpm = 1; u16 dpm_idx; Mpi2DriverMap0Entry_t *dpm_entry; - if (!sc->is_dpm_enable) - return; + /* + * Depending on the reason code, update the missing count. Always set + * the init_complete flag when here, so just do it first. That flag is + * used for volumes to make sure that the DPM entry has been updated. + * When a volume is deleted, clear the map entry's IN_USE flag so that + * the entry can be used again if another volume is created. Also clear + * its dev_handle entry so that other functions can't find this volume + * by the handle, since it's not defined any longer. + */ mt_entry = &sc->mapping_table[map_idx]; - if (reason == MPI2_EVENT_IR_CHANGE_RC_ADDED) { + mt_entry->init_complete = 1; + if ((reason == MPI2_EVENT_IR_CHANGE_RC_ADDED) || + (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED)) { mt_entry->missing_count = 0; - } else if (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED) { - mt_entry->missing_count = 0; - mt_entry->init_complete = 0; - } else if ((reason == MPI2_EVENT_IR_CHANGE_RC_REMOVED) || - (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED)) { - if (!mt_entry->init_complete) { - if (mt_entry->missing_count < MPS_MAX_MISSING_COUNT) - mt_entry->missing_count++; - else - mt_entry->init_complete = 1; - } - if (!mt_entry->missing_count) + } else if (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED) { + if (mt_entry->missing_count < MPS_MAX_MISSING_COUNT) mt_entry->missing_count++; + + mt_entry->device_info &= ~MPS_MAP_IN_USE; mt_entry->dev_handle = 0; } + /* + * If persistent mapping is enabled, update the DPM with the new missing + * count for the volume. If the DPM index is bad, get a free one. If + * it's bad for a volume that's being deleted do nothing because that + * volume doesn't have a DPM entry. + */ + if (!sc->is_dpm_enable) + return; dpm_idx = mt_entry->dpm_entry_num; if (dpm_idx == MPS_DPM_BAD_IDX) { - if ((reason == MPI2_EVENT_IR_CHANGE_RC_ADDED) || - (reason == MPI2_EVENT_IR_CHANGE_RC_REMOVED)) - dpm_idx = _mapping_get_dpm_idx_from_id(sc, - mt_entry->physical_id, 0); - else if (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED) + if (reason == MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED) + { + mps_dprint(sc, MPS_MAPPING, "%s: Volume being deleted " + "is not in DPM so DPM missing count will not be " + "updated.\n", __func__); return; + } } + if (dpm_idx == MPS_DPM_BAD_IDX) + dpm_idx = _mapping_get_free_dpm_idx(sc); + + /* + * Got the DPM entry for the volume or found a free DPM entry if this is + * a new volume. Check if the current information is outdated. + */ if (dpm_idx != MPS_DPM_BAD_IDX) { dpm_entry = (Mpi2DriverMap0Entry_t *)((u8 *)sc->dpm_pg0 + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); @@ -607,17 +709,25 @@ missing_cnt = dpm_entry->MappingInformation & MPI2_DRVMAP0_MAPINFO_MISSING_MASK; if ((mt_entry->physical_id == - le64toh((u64)dpm_entry->PhysicalIdentifier.High | - dpm_entry->PhysicalIdentifier.Low)) && (missing_cnt == - mt_entry->missing_count)) - mt_entry->init_complete = 1; - } else { - dpm_idx = _mapping_get_free_dpm_idx(sc); - mt_entry->init_complete = 0; + le64toh(((u64)dpm_entry->PhysicalIdentifier.High << 32) | + (u64)dpm_entry->PhysicalIdentifier.Low)) && (missing_cnt == + mt_entry->missing_count)) { + mps_dprint(sc, MPS_MAPPING, "%s: DPM entry for volume " + "with target ID %d does not require an update.\n", + __func__, mt_entry->id); + update_dpm = 0; + } } - if ((dpm_idx != MPS_DPM_BAD_IDX) && !mt_entry->init_complete) { - mt_entry->init_complete = 1; + /* + * Update the volume's persistent info if it's new or the ID or missing + * count has changed. If a good DPM index has not been found by now, + * there is no space left in the DPM table. + */ + if ((dpm_idx != MPS_DPM_BAD_IDX) && update_dpm) { + mps_dprint(sc, MPS_MAPPING, "%s: Update DPM entry for volume " + "with target ID %d.\n", __func__, mt_entry->id); + mt_entry->dpm_entry_num = dpm_idx; dpm_entry = (Mpi2DriverMap0Entry_t *)((u8 *)sc->dpm_pg0 + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); @@ -633,44 +743,47 @@ sc->dpm_flush_entry[dpm_idx] = 1; sc->dpm_entry_used[dpm_idx] = 1; } else if (dpm_idx == MPS_DPM_BAD_IDX) { - printf("%s: no space to add entry in DPM table\n", __func__); - mt_entry->init_complete = 1; + mps_dprint(sc, MPS_INFO | MPS_MAPPING, "%s: No space to add an " + "entry in the DPM table for volume with target ID %d.\n", + __func__, mt_entry->id); } } /** - * _mapping_add_to_removal_table - mark an entry for removal + * _mapping_add_to_removal_table - add DPM index to the removal table * @sc: per adapter object - * @handle: Handle of enclosures/device/volume + * @dpm_idx: Index of DPM entry to remove * - * Adds the handle or DPM entry number in removal table. + * Adds a DPM entry number to the removal table. * * Returns nothing. */ static void -_mapping_add_to_removal_table(struct mps_softc *sc, u16 handle, - u16 dpm_idx) +_mapping_add_to_removal_table(struct mps_softc *sc, u16 dpm_idx) { struct map_removal_table *remove_entry; u32 i; - u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags); + /* + * This is only used to remove entries from the DPM in the controller. + * If DPM is not enabled, just return. + */ + if (!sc->is_dpm_enable) + return; + + /* + * Find the first available removal_table entry and add the new entry + * there. + */ remove_entry = sc->removal_table; for (i = 0; i < sc->max_devices; i++, remove_entry++) { - if (remove_entry->dev_handle || remove_entry->dpm_entry_num != - MPS_DPM_BAD_IDX) + if (remove_entry->dpm_entry_num != MPS_DPM_BAD_IDX) continue; - if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == - MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { - if (dpm_idx) - remove_entry->dpm_entry_num = dpm_idx; - if (remove_entry->dpm_entry_num == MPS_DPM_BAD_IDX) - remove_entry->dev_handle = handle; - } else if ((ioc_pg8_flags & - MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == - MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) - remove_entry->dev_handle = handle; + + mps_dprint(sc, MPS_MAPPING, "%s: Adding DPM entry %d to table " + "for removal.\n", __func__, dpm_idx); + remove_entry->dpm_entry_num = dpm_idx; break; } @@ -681,8 +794,13 @@ * @sc: per adapter object * @topo_change: Topology change event entry * - * Search through the topology change list and if any device is found not - * responding it's associated map table entry and DPM entry is updated + * Increment the missing count in the mapping table for a device that is not + * responding. If Persitent Mapping is used, increment the DPM entry as well. + * Currently, this function only increments the missing count if the device + * goes missing, so after initialization has completed. This means that the + * missing count can only go from 0 to 1 here. The missing count is incremented + * during initialization as well, so that's where a target's missing count can + * go past 1. * * Returns nothing. */ @@ -706,34 +824,45 @@ dev_handle); phy_change->is_processed = 1; if (map_idx == MPS_MAPTABLE_BAD_IDX) { - printf("%s: device is already removed from mapping " - "table\n", __func__); + mps_dprint(sc, MPS_INFO | MPS_MAPPING, "%s: device is " + "already removed from mapping table\n", __func__); continue; } mt_entry = &sc->mapping_table[map_idx]; - if (!mt_entry->init_complete) { - if (mt_entry->missing_count < MPS_MAX_MISSING_COUNT) - mt_entry->missing_count++; - else - mt_entry->init_complete = 1; - } - if (!mt_entry->missing_count) + if (mt_entry->missing_count < MPS_MAX_MISSING_COUNT) mt_entry->missing_count++; - _mapping_add_to_removal_table(sc, mt_entry->dev_handle, 0); - mt_entry->dev_handle = 0; + /* + * When using Enc/Slot mapping, when a device is removed, it's + * mapping table information should be cleared. Otherwise, the + * target ID will be incorrect if this same device is re-added + * to a different slot. + */ + if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == + MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { + _mapping_clear_map_entry(mt_entry); + } + + /* + * When using device mapping, update the missing count in the + * DPM entry, but only if the missing count has changed. + */ if (((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) && - sc->is_dpm_enable && !mt_entry->init_complete && + sc->is_dpm_enable && mt_entry->dpm_entry_num != MPS_DPM_BAD_IDX) { dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); dpm_entry += mt_entry->dpm_entry_num; - dpm_entry->MappingInformation = mt_entry->missing_count; - sc->dpm_flush_entry[mt_entry->dpm_entry_num] = 1; + if (dpm_entry->MappingInformation != + mt_entry->missing_count) { + dpm_entry->MappingInformation = + mt_entry->missing_count; + sc->dpm_flush_entry[mt_entry->dpm_entry_num] = + 1; + } } - mt_entry->init_complete = 1; } } @@ -766,6 +895,10 @@ vol_mapping_flags = le16toh(sc->ioc_pg8.IRVolumeMappingFlags) & MPI2_IOCPAGE8_IRFLAGS_MASK_VOLUME_MAPPING_MODE; + /* + * The end of the mapping table depends on where volumes are kept, if + * IR is enabled. + */ if (!sc->ir_firmware) end_of_table = sc->max_devices; else if (vol_mapping_flags == MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING) @@ -773,6 +906,17 @@ else end_of_table = sc->max_devices - sc->max_volumes; + /* + * The skip_count is the number of entries that are reserved at the + * beginning of the mapping table. But, it does not include the number + * of Physical IDs that are reserved for direct attached devices. Look + * through the mapping table after these reserved entries to see if + * the devices for this enclosure are already mapped. The PHY bit check + * is used to make sure that at least one PHY bit is common between the + * enclosure and the device that is already mapped. + */ + mps_dprint(sc, MPS_MAPPING, "%s: Looking for space in the mapping " + "table for added enclosure.\n", __func__); for (map_idx = (max_num_phy_ids + skip_count); map_idx < end_of_table; map_idx++) { mt_entry = &sc->mapping_table[map_idx]; @@ -782,11 +926,22 @@ num_found += 1; if (num_found == et_entry->num_slots) { start_idx = (map_idx - num_found) + 1; + mps_dprint(sc, MPS_MAPPING, "%s: Found space " + "in the mapping for enclosure at map index " + "%d.\n", __func__, start_idx); return start_idx; } } else num_found = 0; } + + + /* + * If the enclosure's devices are not mapped already, look for + * contiguous entries in the mapping table that are not reserved. If + * enough entries are found, return the starting index for that space. + */ + num_found = 0; for (map_idx = (max_num_phy_ids + skip_count); map_idx < end_of_table; map_idx++) { mt_entry = &sc->mapping_table[map_idx]; @@ -794,40 +949,91 @@ num_found += 1; if (num_found == et_entry->num_slots) { start_idx = (map_idx - num_found) + 1; + mps_dprint(sc, MPS_MAPPING, "%s: Found space " + "in the mapping for enclosure at map index " + "%d.\n", __func__, start_idx); return start_idx; } } else num_found = 0; } + /* + * If here, it means that not enough space in the mapping table was + * found to support this enclosure, so go through the enclosure table to + * see if any enclosure entries have a missing count. If so, get the + * enclosure with the highest missing count and check it to see if there + * is enough space for the new enclosure. + */ while (!done_flag) { enc_idx = _mapping_get_high_missing_et_idx(sc); - if (enc_idx == MPS_ENCTABLE_BAD_IDX) + if (enc_idx == MPS_ENCTABLE_BAD_IDX) { + mps_dprint(sc, MPS_MAPPING, "%s: Not enough space was " + "found in the mapping for the added enclosure.\n", + __func__); return MPS_MAPTABLE_BAD_IDX; + } + + /* + * Found a missing enclosure. Set the skip_search flag so this + * enclosure is not checked again for a high missing count if + * the loop continues. This way, all missing enclosures can + * have their space added together to find enough space in the + * mapping table for the added enclosure. The space must be + * contiguous. + */ + mps_dprint(sc, MPS_MAPPING, "%s: Space from a missing " + "enclosure was found.\n", __func__); enc_entry = &sc->enclosure_table[enc_idx]; - /*VSP FIXME*/ enc_entry->skip_search = 1; + + /* + * Unmark all of the missing enclosure's device's reserved + * space. These will be remarked as reserved if this missing + * enclosure's space is not used. + */ + mps_dprint(sc, MPS_MAPPING, "%s: Clear the reserved flag for " + "all of the map entries for the enclosure.\n", __func__); mt_entry = &sc->mapping_table[enc_entry->start_index]; for (map_idx = enc_entry->start_index; map_idx < (enc_entry->start_index + enc_entry->num_slots); map_idx++, mt_entry++) - mt_entry->device_info &= ~MPS_DEV_RESERVED; + mt_entry->device_info &= ~MPS_DEV_RESERVED; + + /* + * Now that space has been unreserved, check again to see if + * enough space is available for the new enclosure. + */ + mps_dprint(sc, MPS_MAPPING, "%s: Check if new mapping space is " + "enough for the new enclosure.\n", __func__); found_space = 0; - for (map_idx = (max_num_phy_ids + - skip_count); map_idx < end_of_table; map_idx++) { + num_found = 0; + for (map_idx = (max_num_phy_ids + skip_count); + map_idx < end_of_table; map_idx++) { mt_entry = &sc->mapping_table[map_idx]; if (!(mt_entry->device_info & MPS_DEV_RESERVED)) { num_found += 1; if (num_found == et_entry->num_slots) { start_idx = (map_idx - num_found) + 1; found_space = 1; + break; } } else num_found = 0; } - if (!found_space) continue; + + /* + * If enough space was found, all of the missing enclosures that + * will be used for the new enclosure must be added to the + * removal table. Then all mappings for the enclosure's devices + * and for the enclosure itself need to be cleared. There may be + * more than one enclosure to add to the removal table and + * clear. + */ + mps_dprint(sc, MPS_MAPPING, "%s: Found space in the mapping " + "for enclosure at map index %d.\n", __func__, start_idx); for (map_idx = start_idx; map_idx < (start_idx + num_found); map_idx++) { enc_entry = sc->enclosure_table; @@ -838,26 +1044,38 @@ enc_entry->num_slots)) continue; if (!enc_entry->removal_flag) { + mps_dprint(sc, MPS_MAPPING, "%s: " + "Enclosure %d will be removed from " + "the mapping table.\n", __func__, + enc_idx); enc_entry->removal_flag = 1; - _mapping_add_to_removal_table(sc, 0, + _mapping_add_to_removal_table(sc, enc_entry->dpm_entry_num); } mt_entry = &sc->mapping_table[map_idx]; - if (mt_entry->device_info & - MPS_MAP_IN_USE) { - _mapping_add_to_removal_table(sc, - mt_entry->dev_handle, 0); - _mapping_clear_map_entry(mt_entry); - } + _mapping_clear_map_entry(mt_entry); if (map_idx == (enc_entry->start_index + enc_entry->num_slots - 1)) _mapping_clear_enc_entry(et_entry); } } + + /* + * During the search for space for this enclosure, some entries + * in the mapping table may have been unreserved. Go back and + * change all of these to reserved again. Only the enclosures + * with the removal_flag set should be left as unreserved. The + * skip_search flag needs to be cleared as well so that the + * enclosure's space will be looked at the next time space is + * needed. + */ enc_entry = sc->enclosure_table; for (enc_idx = 0; enc_idx < sc->num_enc_table_entries; enc_idx++, enc_entry++) { if (!enc_entry->removal_flag) { + mps_dprint(sc, MPS_MAPPING, "%s: Reset the " + "reserved flag for all of the map entries " + "for enclosure %d.\n", __func__, enc_idx); mt_entry = &sc->mapping_table[enc_entry-> start_index]; for (map_idx = enc_entry->start_index; map_idx < @@ -905,6 +1123,7 @@ if (phy_change->is_processed || !phy_change->dev_handle || phy_change->reason != MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED) continue; + if (mps_config_get_sas_device_pg0(sc, &mpi_reply, &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, phy_change->dev_handle)) { @@ -918,9 +1137,9 @@ * when the system is shutdown. */ device_info = le32toh(sas_device_pg0.DeviceInfo); - sas_address = sas_device_pg0.SASAddress.High; + sas_address = le32toh(sas_device_pg0.SASAddress.High); sas_address = (sas_address << 32) | - sas_device_pg0.SASAddress.Low; + le32toh(sas_device_pg0.SASAddress.Low); if ((device_info & MPI2_SAS_DEVICE_INFO_END_DEVICE) && (device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)) { rc = mpssas_get_sas_address_for_sata_disk(sc, @@ -939,18 +1158,29 @@ phy_change->physical_id = sas_address; phy_change->slot = le16toh(sas_device_pg0.Slot); - phy_change->device_info = le32toh(sas_device_pg0.DeviceInfo); + phy_change->device_info = device_info; + /* + * When using Enc/Slot mapping, if this device is an enclosure + * make sure that all of its slots can fit into the mapping + * table. + */ if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { + /* + * The enclosure should already be in the enclosure + * table due to the Enclosure Add event. If not, just + * continue, nothing can be done. + */ enc_idx = _mapping_get_enc_idx_from_handle(sc, topo_change->enc_handle); if (enc_idx == MPS_ENCTABLE_BAD_IDX) { phy_change->is_processed = 1; - mps_dprint(sc, MPS_MAPPING, "%s: failed to add " - "the device with handle 0x%04x because the " - "enclosure is not in the mapping table\n", - __func__, phy_change->dev_handle); + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because the enclosure is not in " + "the mapping table\n", __func__, + phy_change->dev_handle); continue; } if (!((phy_change->device_info & @@ -963,8 +1193,20 @@ continue; } et_entry = &sc->enclosure_table[enc_idx]; + + /* + * If the enclosure already has a start_index, it's been + * mapped, so go to the next Topo change. + */ if (et_entry->start_index != MPS_MAPTABLE_BAD_IDX) continue; + + /* + * If the Expander Handle is 0, the devices are direct + * attached. In that case, the start_index must be just + * after the reserved entries. Otherwise, find space in + * the mapping table for the enclosure's devices. + */ if (!topo_change->exp_handle) { map_idx = sc->num_rsvd_entries; et_entry->start_index = map_idx; @@ -972,8 +1214,26 @@ map_idx = _mapping_find_enc_map_space(sc, et_entry); et_entry->start_index = map_idx; + + /* + * If space cannot be found to hold all of the + * enclosure's devices in the mapping table, + * there's no need to continue checking the + * other devices in this event. Set all of the + * phy_details for this event (if the change is + * for an add) as already processed because none + * of these devices can be added to the mapping + * table. + */ if (et_entry->start_index == MPS_MAPTABLE_BAD_IDX) { + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, + "%s: failed to add the enclosure " + "with ID 0x%016jx because there is " + "no free space available in the " + "mapping table for all of the " + "enclosure's devices.\n", __func__, + (uintmax_t)et_entry->enclosure_id); phy_change->is_processed = 1; for (phy_idx = 0; phy_idx < topo_change->num_entries; @@ -989,12 +1249,22 @@ break; } } + + /* + * Found space in the mapping table for this enclosure. + * Initialize each mapping table entry for the + * enclosure. + */ + mps_dprint(sc, MPS_MAPPING, "%s: Initialize %d map " + "entries for the enclosure, starting at map index " + " %d.\n", __func__, et_entry->num_slots, map_idx); mt_entry = &sc->mapping_table[map_idx]; for (index = map_idx; index < (et_entry->num_slots + map_idx); index++, mt_entry++) { mt_entry->device_info = MPS_DEV_RESERVED; mt_entry->physical_id = et_entry->enclosure_id; mt_entry->phy_bits = et_entry->phy_bits; + mt_entry->missing_count = 0; } } } @@ -1014,6 +1284,7 @@ struct dev_mapping_table *mt_entry; u16 slots = et_entry->num_slots, map_idx; u32 start_idx = et_entry->start_index; + if (start_idx != MPS_MAPTABLE_BAD_IDX) { mt_entry = &sc->mapping_table[start_idx]; for (map_idx = 0; map_idx < slots; map_idx++, mt_entry++) @@ -1063,6 +1334,13 @@ } } } + + /* + * When using Enc/Slot mapping, if a new enclosure was added and old + * enclosure space was needed, the enclosure table may now have gaps + * that need to be closed. All enclosure mappings need to be contiguous + * so that space can be reused correctly if available. + */ if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { num_entries = sc->num_enc_table_entries; @@ -1105,8 +1383,8 @@ * @sc: per adapter object * @topo_change: Topology change event entry * - * Search through the topology change event list and updates map table, - * enclosure table and DPM pages for for the newly added devices. + * Search through the topology change event list and update map table, + * enclosure table and DPM pages for the newly added devices. * * Returns nothing */ @@ -1143,30 +1421,41 @@ (sc, topo_change->enc_handle); if (enc_idx == MPS_ENCTABLE_BAD_IDX) { phy_change->is_processed = 1; - printf("%s: failed to add the device with " - "handle 0x%04x because the enclosure is " - "not in the mapping table\n", __func__, + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because the enclosure is not in " + "the mapping table\n", __func__, phy_change->dev_handle); continue; } + + /* + * If the enclosure's start_index is BAD here, it means + * that there is no room in the mapping table to cover + * all of the devices that could be in the enclosure. + * There's no reason to process any of the devices for + * this enclosure since they can't be mapped. + */ et_entry = &sc->enclosure_table[enc_idx]; if (et_entry->start_index == MPS_MAPTABLE_BAD_IDX) { phy_change->is_processed = 1; - if (!sc->mt_full_retry) { - sc->mt_add_device_failed = 1; - continue; - } - printf("%s: failed to add the device with " - "handle 0x%04x because there is no free " - "space available in the mapping table\n", + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because there is no free space " + "available in the mapping table\n", __func__, phy_change->dev_handle); continue; } + + /* + * Add this device to the mapping table at the correct + * offset where space was found to map the enclosure. + * Then setup the DPM entry information if being used. + */ map_idx = et_entry->start_index + phy_change->slot - et_entry->start_slot; mt_entry = &sc->mapping_table[map_idx]; mt_entry->physical_id = phy_change->physical_id; - mt_entry->channel = 0; mt_entry->id = map_idx; mt_entry->dev_handle = phy_change->dev_handle; mt_entry->missing_count = 0; @@ -1193,28 +1482,28 @@ et_entry->enclosure_id); dpm_entry-> PhysicalIdentifier.High = - ( et_entry->enclosure_id + (et_entry->enclosure_id >> 32); dpm_entry->DeviceIndex = (U16)et_entry->start_index; dpm_entry->MappingInformation = - et_entry->num_slots; + et_entry->num_slots; dpm_entry->MappingInformation <<= map_shift; dpm_entry->PhysicalBitsMapping = et_entry->phy_bits; et_entry->dpm_entry_num = dpm_idx; - /* FIXME Do I need to set the dpm_idxin mt_entry too */ sc->dpm_entry_used[dpm_idx] = 1; sc->dpm_flush_entry[dpm_idx] = 1; phy_change->is_processed = 1; } else { phy_change->is_processed = 1; - mps_dprint(sc, MPS_INFO, "%s: " - "failed to add the device " - "with handle 0x%04x to " + mps_dprint(sc, MPS_ERROR | + MPS_MAPPING, "%s: failed " + "to add the device with " + "handle 0x%04x to " "persistent table because " "there is no free space " "available\n", __func__, @@ -1225,11 +1514,20 @@ mt_entry->dpm_entry_num = dpm_idx; } } - /* FIXME Why not mt_entry too? */ et_entry->init_complete = 1; } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + + /* + * Get the mapping table index for this device. If it's + * not in the mapping table yet, find a free entry if + * one is available. If there are no free entries, look + * for the entry that has the highest missing count. If + * none of that works to find an entry in the mapping + * table, there is a problem. Log a message and just + * continue on. + */ map_idx = _mapping_get_mt_idx_from_id (sc, phy_change->physical_id); if (map_idx == MPS_MAPTABLE_BAD_IDX) { @@ -1239,23 +1537,24 @@ map_idx = _mapping_get_free_mt_idx(sc, search_idx); } + + /* + * If an entry will be used that has a missing device, + * clear its entry from the DPM in the controller. + */ if (map_idx == MPS_MAPTABLE_BAD_IDX) { map_idx = _mapping_get_high_missing_mt_idx(sc); if (map_idx != MPS_MAPTABLE_BAD_IDX) { mt_entry = &sc->mapping_table[map_idx]; - if (mt_entry->dev_handle) { - _mapping_add_to_removal_table - (sc, mt_entry->dev_handle, - 0); - is_removed = 1; - } + _mapping_add_to_removal_table(sc, + mt_entry->dpm_entry_num); + is_removed = 1; mt_entry->init_complete = 0; } } if (map_idx != MPS_MAPTABLE_BAD_IDX) { mt_entry = &sc->mapping_table[map_idx]; mt_entry->physical_id = phy_change->physical_id; - mt_entry->channel = 0; mt_entry->id = map_idx; mt_entry->dev_handle = phy_change->dev_handle; mt_entry->missing_count = 0; @@ -1263,13 +1562,10 @@ | (MPS_DEV_RESERVED | MPS_MAP_IN_USE); } else { phy_change->is_processed = 1; - if (!sc->mt_full_retry) { - sc->mt_add_device_failed = 1; - continue; - } - printf("%s: failed to add the device with " - "handle 0x%04x because there is no free " - "space available in the mapping table\n", + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: " + "failed to add the device with handle " + "0x%04x because there is no free space " + "available in the mapping table\n", __func__, phy_change->dev_handle); continue; } @@ -1287,16 +1583,24 @@ PhysicalIdentifier.High; temp64_var = (temp64_var << 32) | dpm_entry->PhysicalIdentifier.Low; + + /* + * If the Mapping Table's info is not + * the same as the DPM entry, clear the + * init_complete flag so that it's + * updated. + */ if ((mt_entry->physical_id == temp64_var) && !missing_cnt) mt_entry->init_complete = 1; + else + mt_entry->init_complete = 0; } else { dpm_idx = _mapping_get_free_dpm_idx(sc); mt_entry->init_complete = 0; } if (dpm_idx != MPS_DPM_BAD_IDX && !mt_entry->init_complete) { - mt_entry->init_complete = 1; mt_entry->dpm_entry_num = dpm_idx; dpm_entry = (Mpi2DriverMap0Entry_t *) ((u8 *)sc->dpm_pg0 + hdr_sz); @@ -1313,14 +1617,13 @@ sc->dpm_flush_entry[dpm_idx] = 1; phy_change->is_processed = 1; } else if (dpm_idx == MPS_DPM_BAD_IDX) { - phy_change->is_processed = 1; - mps_dprint(sc, MPS_INFO, "%s: " - "failed to add the device " - "with handle 0x%04x to " - "persistent table because " - "there is no free space " - "available\n", __func__, - phy_change->dev_handle); + phy_change->is_processed = 1; + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, + "%s: failed to add the device with " + "handle 0x%04x to persistent table " + "because there is no free space " + "available\n", __func__, + phy_change->dev_handle); } } mt_entry->init_complete = 1; @@ -1363,10 +1666,13 @@ memcpy(&config_page.Entry, (u8 *)dpm_entry, sizeof(Mpi2DriverMap0Entry_t)); /* TODO-How to handle failed writes? */ + mps_dprint(sc, MPS_MAPPING, "%s: Flushing DPM entry %d.\n", + __func__, entry_num); if (mps_config_set_dpm_pg0(sc, &mpi_reply, &config_page, entry_num)) { - printf("%s: write of dpm entry %d for device failed\n", - __func__, entry_num); + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: Flush of " + "DPM entry %d for device failed\n", __func__, + entry_num); } else sc->dpm_flush_entry[entry_num] = 0; dpm_entry->MappingInformation = le16toh(dpm_entry-> @@ -1456,7 +1762,6 @@ free(sc->dpm_pg0, M_MPT2); } - static void _mapping_process_dpm_pg0(struct mps_softc *sc) { @@ -1471,9 +1776,20 @@ u64 physical_id; u32 phy_bits = 0; + /* + * start_idx and end_idx are only used for IR. + */ if (sc->ir_firmware) _mapping_get_ir_maprange(sc, &start_idx, &end_idx); + /* + * Look through all of the DPM entries that were read from the + * controller and copy them over to the driver's internal table if they + * have a non-zero ID. At this point, any ID with a value of 0 would be + * invalid, so don't copy it. + */ + mps_dprint(sc, MPS_MAPPING, "%s: Start copy of %d DPM entries into the " + "mapping table.\n", __func__, sc->max_dpm_entries); dpm_entry = (Mpi2DriverMap0Entry_t *) ((uint8_t *) sc->dpm_pg0 + sizeof(MPI2_CONFIG_EXTENDED_PAGE_HEADER)); for (entry_num = 0; entry_num < sc->max_dpm_entries; entry_num++, @@ -1492,13 +1808,20 @@ MPI2_DRVMAP0_MAPINFO_MISSING_MASK; dev_idx = le16toh(dpm_entry->DeviceIndex); phy_bits = le32toh(dpm_entry->PhysicalBitsMapping); + + /* + * Volumes are at special locations in the mapping table so + * account for that. Volume mapping table entries do not depend + * on the type of mapping, so continue the loop after adding + * volumes to the mapping table. + */ if (sc->ir_firmware && (dev_idx >= start_idx) && (dev_idx <= end_idx)) { mt_entry = &sc->mapping_table[dev_idx]; - mt_entry->physical_id = dpm_entry->PhysicalIdentifier.High; + mt_entry->physical_id = + dpm_entry->PhysicalIdentifier.High; mt_entry->physical_id = (mt_entry->physical_id << 32) | dpm_entry->PhysicalIdentifier.Low; - mt_entry->channel = MPS_RAID_CHANNEL; mt_entry->id = dev_idx; mt_entry->missing_count = missing_cnt; mt_entry->dpm_entry_num = entry_num; @@ -1507,7 +1830,16 @@ } if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { - if (dev_idx < (sc->num_rsvd_entries + + + /* + * The dev_idx for an enclosure is the start index. If + * the start index is within the controller's default + * enclosure area, set the number of slots for this + * enclosure to the max allowed. Otherwise, it should be + * a normal enclosure and the number of slots is in the + * DPM entry's Mapping Information. + */ + if (dev_idx < (sc->num_rsvd_entries + max_num_phy_ids)) { slot_id = 0; if (ioc_pg8_flags & @@ -1522,8 +1854,9 @@ } enc_idx = sc->num_enc_table_entries; if (enc_idx >= sc->max_enclosures) { - printf("%s: enclosure entries exceed max " - "enclosures of %d\n", __func__, + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: " + "Number of enclosure entries in DPM exceed " + "the max allowed of %d.\n", __func__, sc->max_enclosures); break; } @@ -1539,21 +1872,32 @@ et_entry->missing_count = missing_cnt; et_entry->phy_bits = phy_bits; + /* + * Initialize all entries for this enclosure in the + * mapping table and mark them as reserved. The actual + * devices have not been processed yet but when they are + * they will use these entries. If an entry is found + * that already has a valid DPM index, the mapping table + * is corrupt. This can happen if the mapping type is + * changed without clearing all of the DPM entries in + * the controller. + */ mt_entry = &sc->mapping_table[dev_idx]; for (map_idx = dev_idx; map_idx < (dev_idx + num_slots); map_idx++, mt_entry++) { if (mt_entry->dpm_entry_num != MPS_DPM_BAD_IDX) { - printf("%s: conflict in mapping table " - "for enclosure %d\n", __func__, + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, + "%s: Conflict in mapping table for " + " enclosure %d\n", __func__, enc_idx); break; } - physical_id = dpm_entry->PhysicalIdentifier.High; + physical_id = + dpm_entry->PhysicalIdentifier.High; mt_entry->physical_id = (physical_id << 32) | dpm_entry->PhysicalIdentifier.Low; mt_entry->phy_bits = phy_bits; - mt_entry->channel = 0; mt_entry->id = dev_idx; mt_entry->dpm_entry_num = entry_num; mt_entry->missing_count = missing_cnt; @@ -1562,18 +1906,24 @@ } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + + /* + * Device mapping, so simply copy the DPM entries to the + * mapping table, but check for a corrupt mapping table + * (as described above in Enc/Slot mapping). + */ map_idx = dev_idx; mt_entry = &sc->mapping_table[map_idx]; if (mt_entry->dpm_entry_num != MPS_DPM_BAD_IDX) { - printf("%s: conflict in mapping table for " - "device %d\n", __func__, map_idx); + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: " + "Conflict in mapping table for device %d\n", + __func__, map_idx); break; } physical_id = dpm_entry->PhysicalIdentifier.High; mt_entry->physical_id = (physical_id << 32) | dpm_entry->PhysicalIdentifier.Low; mt_entry->phy_bits = phy_bits; - mt_entry->channel = 0; mt_entry->id = dev_idx; mt_entry->missing_count = missing_cnt; mt_entry->dpm_entry_num = entry_num; @@ -1585,43 +1935,91 @@ /* * mps_mapping_check_devices - start of the day check for device availabilty * @sc: per adapter object - * @sleep_flag: Flag indicating whether this function can sleep or not * * Returns nothing. */ void -mps_mapping_check_devices(struct mps_softc *sc, int sleep_flag) +mps_mapping_check_devices(void *data) { u32 i; -/* u32 cntdn, i; - u32 timeout = 60;*/ struct dev_mapping_table *mt_entry; + struct mps_softc *sc = (struct mps_softc *)data; u16 ioc_pg8_flags = le16toh(sc->ioc_pg8.Flags); struct enc_mapping_table *et_entry; - u32 start_idx, end_idx; + u32 start_idx = 0, end_idx = 0; + u8 stop_device_checks = 0; - /* We need to ucomment this when this function is called - * from the port enable complete */ -#if 0 + MPS_FUNCTRACE(sc); + + /* + * Clear this flag so that this function is never called again except + * within this function if the check needs to be done again. The + * purpose is to check for missing devices that are currently in the + * mapping table so do this only at driver init after discovery. + */ sc->track_mapping_events = 0; - cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout; - do { - if (!sc->pending_map_events) - break; - if (sleep_flag == CAN_SLEEP) - pause("mps_pause", (hz/1000));/* 1msec sleep */ - else - DELAY(500); /* 500 useconds delay */ - } while (--cntdn); + /* + * callout synchronization + * This is used to prevent race conditions for the callout. + */ + mps_dprint(sc, MPS_MAPPING, "%s: Start check for missing devices.\n", + __func__); + mtx_assert(&sc->mps_mtx, MA_OWNED); + if ((callout_pending(&sc->device_check_callout)) || + (!callout_active(&sc->device_check_callout))) { + mps_dprint(sc, MPS_MAPPING, "%s: Device Check Callout is " + "already pending or not active.\n", __func__); + return; + } + callout_deactivate(&sc->device_check_callout); - if (!cntdn) - printf("%s: there are %d" - " pending events after %d seconds of delay\n", - __func__, sc->pending_map_events, timeout); -#endif - sc->pending_map_events = 0; + /* + * Use callout to check if any devices in the mapping table have been + * processed yet. If ALL devices are marked as not init_complete, no + * devices have been processed and mapped. Until devices are mapped + * there's no reason to mark them as missing. Continue resetting this + * callout until devices have been mapped. + */ + if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == + MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { + et_entry = sc->enclosure_table; + for (i = 0; i < sc->num_enc_table_entries; i++, et_entry++) { + if (et_entry->init_complete) { + stop_device_checks = 1; + break; + } + } + } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == + MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + mt_entry = sc->mapping_table; + for (i = 0; i < sc->max_devices; i++, mt_entry++) { + if (mt_entry->init_complete) { + stop_device_checks = 1; + break; + } + } + } + /* + * Setup another callout check after a delay. Keep doing this until + * devices are mapped. + */ + if (!stop_device_checks) { + mps_dprint(sc, MPS_MAPPING, "%s: No devices have been mapped. " + "Reset callout to check again after a %d second delay.\n", + __func__, MPS_MISSING_CHECK_DELAY); + callout_reset(&sc->device_check_callout, + MPS_MISSING_CHECK_DELAY * hz, mps_mapping_check_devices, + sc); + return; + } + mps_dprint(sc, MPS_MAPPING, "%s: Device check complete.\n", __func__); + + /* + * Depending on the mapping type, check if devices have been processed + * and update their missing counts if not processed. + */ if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_ENCLOSURE_SLOT_MAPPING) { et_entry = sc->enclosure_table; @@ -1629,11 +2027,16 @@ if (!et_entry->init_complete) { if (et_entry->missing_count < MPS_MAX_MISSING_COUNT) { + mps_dprint(sc, MPS_MAPPING, "%s: " + "Enclosure %d is missing from the " + "topology. Update its missing " + "count.\n", __func__, i); et_entry->missing_count++; if (et_entry->dpm_entry_num != - MPS_DPM_BAD_IDX) + MPS_DPM_BAD_IDX) { _mapping_commit_enc_entry(sc, et_entry); + } } et_entry->init_complete = 1; } @@ -1642,70 +2045,46 @@ return; _mapping_get_ir_maprange(sc, &start_idx, &end_idx); mt_entry = &sc->mapping_table[start_idx]; - for (i = start_idx; i < (end_idx + 1); i++, mt_entry++) { - if (mt_entry->device_info & MPS_DEV_RESERVED - && !mt_entry->physical_id) - mt_entry->init_complete = 1; - else if (mt_entry->device_info & MPS_DEV_RESERVED) { - if (!mt_entry->init_complete) { - if (mt_entry->missing_count < - MPS_MAX_MISSING_COUNT) { - mt_entry->missing_count++; - if (mt_entry->dpm_entry_num != - MPS_DPM_BAD_IDX) - _mapping_commit_map_entry(sc, - mt_entry); - } - mt_entry->init_complete = 1; - } - } - } } else if ((ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_MASK_MAPPING_MODE) == MPI2_IOCPAGE8_FLAGS_DEVICE_PERSISTENCE_MAPPING) { + start_idx = 0; + end_idx = sc->max_devices - 1; mt_entry = sc->mapping_table; - for (i = 0; i < sc->max_devices; i++, mt_entry++) { - if (mt_entry->device_info & MPS_DEV_RESERVED - && !mt_entry->physical_id) - mt_entry->init_complete = 1; - else if (mt_entry->device_info & MPS_DEV_RESERVED) { - if (!mt_entry->init_complete) { - if (mt_entry->missing_count < - MPS_MAX_MISSING_COUNT) { - mt_entry->missing_count++; - if (mt_entry->dpm_entry_num != - MPS_DPM_BAD_IDX) + } + + /* + * The start and end indices have been set above according to the + * mapping type. Go through these mappings and update any entries that + * do not have the init_complete flag set, which means they are missing. + */ + if (end_idx == 0) + return; + for (i = start_idx; i < (end_idx + 1); i++, mt_entry++) { + if (mt_entry->device_info & MPS_DEV_RESERVED + && !mt_entry->physical_id) + mt_entry->init_complete = 1; + else if (mt_entry->device_info & MPS_DEV_RESERVED) { + if (!mt_entry->init_complete) { + mps_dprint(sc, MPS_MAPPING, "%s: Device in " + "mapping table at index %d is missing from " + "topology. Update its missing count.\n", + __func__, i); + if (mt_entry->missing_count < + MPS_MAX_MISSING_COUNT) { + mt_entry->missing_count++; + if (mt_entry->dpm_entry_num != + MPS_DPM_BAD_IDX) { _mapping_commit_map_entry(sc, mt_entry); } - mt_entry->init_complete = 1; } + mt_entry->init_complete = 1; } } } } - /** - * mps_mapping_is_reinit_required - check whether event replay required - * @sc: per adapter object - * - * Checks the per ioc flags and decide whether reinit of events required - * - * Returns 1 for reinit of ioc 0 for not. - */ -int mps_mapping_is_reinit_required(struct mps_softc *sc) -{ - if (!sc->mt_full_retry && sc->mt_add_device_failed) { - sc->mt_full_retry = 1; - sc->mt_add_device_failed = 0; - _mapping_flush_dpm_pages(sc); - return 1; - } - sc->mt_full_retry = 1; - return 0; -} - -/** * mps_mapping_initialize - initialize mapping tables * @sc: per adapter object * @@ -1733,11 +2112,14 @@ sc->pending_map_events = 0; sc->num_enc_table_entries = 0; sc->num_rsvd_entries = 0; - sc->num_channels = 1; sc->max_dpm_entries = sc->ioc_pg8.MaxPersistentEntries; sc->is_dpm_enable = (sc->max_dpm_entries) ? 1 : 0; sc->track_mapping_events = 0; + + mps_dprint(sc, MPS_MAPPING, "%s: Mapping table has a max of %d entries " + "and DPM has a max of %d entries.\n", __func__, sc->max_devices, + sc->max_dpm_entries); if (ioc_pg8_flags & MPI2_IOCPAGE8_FLAGS_DISABLE_PERSISTENT_MAPPING) sc->is_dpm_enable = 0; @@ -1777,8 +2159,8 @@ retry_read_dpm: if (mps_config_get_dpm_pg0(sc, &mpi_reply, sc->dpm_pg0, dpm_pg0_sz)) { - printf("%s: dpm page read failed; disabling dpm\n", - __func__); + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: DPM page " + "read failed.\n", __func__); if (retry_count < 3) { retry_count++; goto retry_read_dpm; @@ -1789,6 +2171,11 @@ if (sc->is_dpm_enable) _mapping_process_dpm_pg0(sc); + else { + mps_dprint(sc, MPS_MAPPING, "%s: DPM processing is disabled. " + "Device mappings will not persist across reboots or " + "resets.\n", __func__); + } sc->track_mapping_events = 1; return 0; @@ -1808,15 +2195,15 @@ } /** - * mps_mapping_get_sas_id - assign a target id for sas device + * mps_mapping_get_tid - return the target id for sas device and handle * @sc: per adapter object * @sas_address: sas address of the device * @handle: device handle * - * Returns valid ID on success or BAD_ID. + * Returns valid target ID on success or BAD_ID. */ unsigned int -mps_mapping_get_sas_id(struct mps_softc *sc, uint64_t sas_address, u16 handle) +mps_mapping_get_tid(struct mps_softc *sc, uint64_t sas_address, u16 handle) { u32 map_idx; struct dev_mapping_table *mt_entry; @@ -1832,38 +2219,39 @@ } /** - * mps_mapping_get_sas_id_from_handle - find a target id in mapping table using + * mps_mapping_get_tid_from_handle - find a target id in mapping table using * only the dev handle. This is just a wrapper function for the local function * _mapping_get_mt_idx_from_handle. * @sc: per adapter object * @handle: device handle * - * Returns valid ID on success or BAD_ID. + * Returns valid target ID on success or BAD_ID. */ unsigned int -mps_mapping_get_sas_id_from_handle(struct mps_softc *sc, u16 handle) +mps_mapping_get_tid_from_handle(struct mps_softc *sc, u16 handle) { return (_mapping_get_mt_idx_from_handle(sc, handle)); } /** - * mps_mapping_get_raid_id - assign a target id for raid device + * mps_mapping_get_raid_tid - return the target id for raid device * @sc: per adapter object * @wwid: world wide identifier for raid volume - * @handle: device handle + * @volHandle: volume device handle * - * Returns valid ID on success or BAD_ID. + * Returns valid target ID on success or BAD_ID. */ unsigned int -mps_mapping_get_raid_id(struct mps_softc *sc, u64 wwid, u16 handle) +mps_mapping_get_raid_tid(struct mps_softc *sc, u64 wwid, u16 volHandle) { - u32 map_idx; + u32 start_idx, end_idx, map_idx; struct dev_mapping_table *mt_entry; - for (map_idx = 0; map_idx < sc->max_devices; map_idx++) { - mt_entry = &sc->mapping_table[map_idx]; - if (mt_entry->dev_handle == handle && mt_entry->physical_id == - wwid) + _mapping_get_ir_maprange(sc, &start_idx, &end_idx); + mt_entry = &sc->mapping_table[start_idx]; + for (map_idx = start_idx; map_idx <= end_idx; map_idx++, mt_entry++) { + if (mt_entry->dev_handle == volHandle && + mt_entry->physical_id == wwid) return mt_entry->id; } @@ -1871,16 +2259,16 @@ } /** - * mps_mapping_get_raid_id_from_handle - find raid device in mapping table + * mps_mapping_get_raid_tid_from_handle - find raid device in mapping table * using only the volume dev handle. This is just a wrapper function for the * local function _mapping_get_ir_mt_idx_from_handle. * @sc: per adapter object * @volHandle: volume device handle * - * Returns valid ID on success or BAD_ID. + * Returns valid target ID on success or BAD_ID. */ unsigned int -mps_mapping_get_raid_id_from_handle(struct mps_softc *sc, u16 volHandle) +mps_mapping_get_raid_tid_from_handle(struct mps_softc *sc, u16 volHandle) { return (_mapping_get_ir_mt_idx_from_handle(sc, volHandle)); } @@ -1914,8 +2302,8 @@ if (event_data->ReasonCode == MPI2_EVENT_SAS_ENCL_RC_ADDED) { if (!event_data->NumSlots) { - printf("%s: enclosure with handle = 0x%x reported 0 " - "slots\n", __func__, + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: Enclosure " + "with handle = 0x%x reported 0 slots.\n", __func__, le16toh(event_data->EnclosureHandle)); goto out; } @@ -1924,13 +2312,22 @@ event_data->EnclosureLogicalID.Low; enc_idx = _mapping_get_enc_idx_from_id(sc, temp64_var, event_data->PhyBits); + + /* + * If the Added enclosure is already in the Enclosure Table, + * make sure that all the the enclosure info is up to date. If + * the enclosure was missing and has just been added back, or if + * the enclosure's Phy Bits have changed, clear the missing + * count and update the Phy Bits in the mapping table and in the + * DPM, if it's being used. + */ if (enc_idx != MPS_ENCTABLE_BAD_IDX) { et_entry = &sc->enclosure_table[enc_idx]; if (et_entry->init_complete && !et_entry->missing_count) { - printf("%s: enclosure %d is already present " - "with handle = 0x%x\n",__func__, enc_idx, - et_entry->enc_handle); + mps_dprint(sc, MPS_MAPPING, "%s: Enclosure %d " + "is already present with handle = 0x%x\n", + __func__, enc_idx, et_entry->enc_handle); goto out; } et_entry->enc_handle = le16toh(event_data-> @@ -1949,8 +2346,7 @@ missing_count = (u8)(dpm_entry->MappingInformation & MPI2_DRVMAP0_MAPINFO_MISSING_MASK); - if (!et_entry->init_complete && ( - missing_count || update_phy_bits)) { + if (missing_count || update_phy_bits) { dpm_entry->MappingInformation = et_entry->num_slots; dpm_entry->MappingInformation @@ -1963,20 +2359,29 @@ } } } else { + /* + * This is a new enclosure that is being added. + * Initialize the Enclosure Table entry. It will be + * finalized when a device is added for the enclosure + * and the enclosure has enough space in the Mapping + * Table to map its devices. + */ enc_idx = sc->num_enc_table_entries; if (enc_idx >= sc->max_enclosures) { - printf("%s: enclosure can not be added; " - "mapping table is full\n", __func__); + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: " + "Enclosure cannot be added to mapping " + "table because it's full.\n", __func__); goto out; } sc->num_enc_table_entries++; et_entry = &sc->enclosure_table[enc_idx]; et_entry->enc_handle = le16toh(event_data-> EnclosureHandle); - et_entry->enclosure_id = event_data-> - EnclosureLogicalID.High; - et_entry->enclosure_id = ( et_entry->enclosure_id << - 32) | event_data->EnclosureLogicalID.Low; + et_entry->enclosure_id = le64toh(event_data-> + EnclosureLogicalID.High); + et_entry->enclosure_id = + ((et_entry->enclosure_id << 32) | + le64toh(event_data->EnclosureLogicalID.Low)); et_entry->start_index = MPS_MAPTABLE_BAD_IDX; et_entry->dpm_entry_num = MPS_DPM_BAD_IDX; et_entry->num_slots = le16toh(event_data->NumSlots); @@ -1986,23 +2391,23 @@ et_entry->init_complete = 1; } else if (event_data->ReasonCode == MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING) { + /* + * An enclosure was removed. Update its missing count and then + * update the DPM entry with the new missing count for the + * enclosure. + */ enc_idx = _mapping_get_enc_idx_from_handle(sc, le16toh(event_data->EnclosureHandle)); if (enc_idx == MPS_ENCTABLE_BAD_IDX) { - printf("%s: cannot unmap enclosure %d because it has " - "already been deleted", __func__, enc_idx); + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, "%s: Cannot " + "unmap enclosure %d because it has already been " + "deleted.\n", __func__, enc_idx); goto out; } et_entry = &sc->enclosure_table[enc_idx]; - if (!et_entry->init_complete) { - if (et_entry->missing_count < MPS_MAX_MISSING_COUNT) - et_entry->missing_count++; - else - et_entry->init_complete = 1; - } - if (!et_entry->missing_count) + if (et_entry->missing_count < MPS_MAX_MISSING_COUNT) et_entry->missing_count++; - if (sc->is_dpm_enable && !et_entry->init_complete && + if (sc->is_dpm_enable && et_entry->dpm_entry_num != MPS_DPM_BAD_IDX) { dpm_entry += et_entry->dpm_entry_num; dpm_entry->MappingInformation = et_entry->num_slots; @@ -2072,66 +2477,6 @@ } /** - * _mapping_check_update_ir_mt_idx - Check and update IR map table index - * @sc: per adapter object - * @event_data: event data payload - * @evt_idx: current event index - * @map_idx: current index and the place holder for new map table index - * @wwid_table: world wide name for volumes in the element table - * - * pass through IR events and find whether any events matches and if so - * tries to find new index if not returns failure - * - * Returns 0 on success and 1 on failure - */ -static int -_mapping_check_update_ir_mt_idx(struct mps_softc *sc, - Mpi2EventDataIrConfigChangeList_t *event_data, int evt_idx, u32 *map_idx, - u64 *wwid_table) -{ - struct dev_mapping_table *mt_entry; - u32 st_idx, end_idx, mt_idx = *map_idx; - u8 match = 0; - Mpi2EventIrConfigElement_t *element; - u16 element_flags; - int i; - - mt_entry = &sc->mapping_table[mt_idx]; - _mapping_get_ir_maprange(sc, &st_idx, &end_idx); -search_again: - match = 0; - for (i = evt_idx + 1; i < event_data->NumElements; i++) { - element = (Mpi2EventIrConfigElement_t *) - &event_data->ConfigElement[i]; - element_flags = le16toh(element->ElementFlags); - if ((element_flags & - MPI2_EVENT_IR_CHANGE_EFLAGS_ELEMENT_TYPE_MASK) != - MPI2_EVENT_IR_CHANGE_EFLAGS_VOLUME_ELEMENT) - continue; - if (element->ReasonCode == MPI2_EVENT_IR_CHANGE_RC_ADDED || - element->ReasonCode == - MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED) { - if (mt_entry->physical_id == wwid_table[i]) { - match = 1; - break; - } - } - } - - if (match) { - do { - mt_idx++; - if (mt_idx > end_idx) - return 1; - mt_entry = &sc->mapping_table[mt_idx]; - } while (mt_entry->device_info & MPS_MAP_IN_USE); - goto search_again; - } - *map_idx = mt_idx; - return 0; -} - -/** * mps_mapping_ir_config_change_event - handle IR config change list events * @sc: per adapter object * @event_data: event data payload @@ -2148,7 +2493,6 @@ u32 map_idx, flags; struct dev_mapping_table *mt_entry; u16 element_flags; - u8 log_full_error = 0; wwid_table = malloc(sizeof(u64) * event_data->NumElements, M_MPT2, M_NOWAIT | M_ZERO); @@ -2156,6 +2500,11 @@ goto out; element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0]; flags = le32toh(event_data->Flags); + + /* + * For volume changes, get the WWID for the volume and put it in a + * table to be used in the processing of the IR change event. + */ for (i = 0; i < event_data->NumElements; i++, element++) { element_flags = le16toh(element->ElementFlags); if ((element->ReasonCode != MPI2_EVENT_IR_CHANGE_RC_ADDED) && @@ -2169,14 +2518,14 @@ MPI2_EVENT_IR_CHANGE_EFLAGS_VOLUME_ELEMENT) { mps_config_get_volume_wwid(sc, le16toh(element->VolDevHandle), &wwid_table[i]); - map_idx = _mapping_get_ir_mt_idx_from_wwid(sc, - wwid_table[i]); - if (map_idx != MPS_MAPTABLE_BAD_IDX) { - mt_entry = &sc->mapping_table[map_idx]; - mt_entry->device_info |= MPS_MAP_IN_USE; - } } } + + /* + * Check the ReasonCode for each element in the IR event and Add/Remove + * Volumes or Physical Disks of Volumes to/from the mapping table. Use + * the WWIDs gotten above in wwid_table. + */ if (flags == MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG) goto out; else { @@ -2190,8 +2539,11 @@ map_idx = _mapping_get_ir_mt_idx_from_wwid (sc, wwid_table[i]); if (map_idx != MPS_MAPTABLE_BAD_IDX) { + /* + * The volume is already in the mapping + * table. Just update it's info. + */ mt_entry = &sc->mapping_table[map_idx]; - mt_entry->channel = MPS_RAID_CHANNEL; mt_entry->id = map_idx; mt_entry->dev_handle = le16toh (element->VolDevHandle); @@ -2201,31 +2553,33 @@ map_idx, element, wwid_table[i]); continue; } + + /* + * Volume is not in mapping table yet. Find a + * free entry in the mapping table at the + * volume mapping locations. If no entries are + * available, this is an error because it means + * there are more volumes than can be mapped + * and that should never happen for volumes. + */ map_idx = _mapping_get_free_ir_mt_idx(sc); if (map_idx == MPS_MAPTABLE_BAD_IDX) - log_full_error = 1; - else if (i < (event_data->NumElements - 1)) { - log_full_error = - _mapping_check_update_ir_mt_idx - (sc, event_data, i, &map_idx, - wwid_table); - } - if (log_full_error) { - printf("%s: no space to add the RAID " - "volume with handle 0x%04x in " - "mapping table\n", __func__, le16toh - (element->VolDevHandle)); + { + mps_dprint(sc, MPS_ERROR | MPS_MAPPING, + "%s: failed to add the volume with " + "handle 0x%04x because there is no " + "free space available in the " + "mapping table\n", __func__, + le16toh(element->VolDevHandle)); continue; } mt_entry = &sc->mapping_table[map_idx]; mt_entry->physical_id = wwid_table[i]; - mt_entry->channel = MPS_RAID_CHANNEL; mt_entry->id = map_idx; mt_entry->dev_handle = le16toh(element-> VolDevHandle); mt_entry->device_info = MPS_DEV_RESERVED | MPS_MAP_IN_USE; - mt_entry->init_complete = 0; _mapping_update_ir_missing_cnt(sc, map_idx, element, wwid_table[i]); } else if (element->ReasonCode == @@ -2233,9 +2587,10 @@ map_idx = _mapping_get_ir_mt_idx_from_wwid(sc, wwid_table[i]); if (map_idx == MPS_MAPTABLE_BAD_IDX) { - printf("%s: failed to remove a volume " - "because it has already been " - "removed\n", __func__); + mps_dprint(sc, MPS_MAPPING,"%s: Failed " + "to remove a volume because it has " + "already been removed.\n", + __func__); continue; } _mapping_update_ir_missing_cnt(sc, map_idx, @@ -2245,9 +2600,10 @@ map_idx = _mapping_get_mt_idx_from_handle(sc, le16toh(element->VolDevHandle)); if (map_idx == MPS_MAPTABLE_BAD_IDX) { - printf("%s: failed to remove volume " - "with handle 0x%04x because it has " - "already been removed\n", __func__, + mps_dprint(sc, MPS_MAPPING,"%s: Failed " + "to remove volume with handle " + "0x%04x because it has already " + "been removed.\n", __func__, le16toh(element->VolDevHandle)); continue; } Index: sys/dev/mps/mps_sas.c =================================================================== --- sys/dev/mps/mps_sas.c +++ sys/dev/mps/mps_sas.c @@ -733,7 +733,7 @@ * of MaxTargets here so that we don't get into trouble later. This * should move into the reinit logic. */ - sassc->maxtargets = sc->facts->MaxTargets; + sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes; sassc->targets = malloc(sizeof(struct mpssas_target) * sassc->maxtargets, M_MPT2, M_WAITOK|M_ZERO); if(!sassc->targets) { @@ -910,12 +910,33 @@ if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING) callout_stop(&sassc->discovery_callout); + /* + * After discovery has completed, check the mapping table for any + * missing devices and update their missing counts. Only do this once + * whenever the driver is initialized so that missing counts aren't + * updated unnecessarily. Note that just because discovery has + * completed doesn't mean that events have been processed yet. The + * check_devices function is a callout timer that checks if ALL devices + * are missing. If so, it will wait a little longer for events to + * complete and keep resetting itself until some device in the mapping + * table is not missing, meaning that event processing has started. + */ + if (sc->track_mapping_events) { + mps_dprint(sc, MPS_XINFO | MPS_MAPPING, "Discovery has " + "completed. Check for missing devices in the mapping " + "table.\n"); + callout_reset(&sc->device_check_callout, + MPS_MISSING_CHECK_DELAY * hz, mps_mapping_check_devices, + sc); + } } static void mpssas_action(struct cam_sim *sim, union ccb *ccb) { struct mpssas_softc *sassc; + u16 volume_mapping_flags; + u16 ioc_pg8_flags; sassc = cam_sim_softc(sim); @@ -942,7 +963,22 @@ cpi->hba_eng_cnt = 0; cpi->max_target = sassc->maxtargets - 1; cpi->max_lun = 255; - cpi->initiator_id = sassc->maxtargets - 1; + + /* + * When setting initiator_id, account for any volume IDs that + * may be after other targets. + */ + ioc_pg8_flags = le16toh(sassc->sc->ioc_pg8.Flags); + volume_mapping_flags = + le16toh(sassc->sc->ioc_pg8.IRVolumeMappingFlags) & + MPI2_IOCPAGE8_IRFLAGS_MASK_VOLUME_MAPPING_MODE; + if (volume_mapping_flags == + MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING) { + cpi->initiator_id = sassc->maxtargets - 1; + } else { + cpi->initiator_id = sassc->maxtargets - + sassc->sc->max_volumes - 1; + } strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); Index: sys/dev/mps/mps_sas_lsi.c =================================================================== --- sys/dev/mps/mps_sas_lsi.c +++ sys/dev/mps/mps_sas_lsi.c @@ -217,9 +217,11 @@ switch (phy->PhyStatus & MPI2_EVENT_SAS_TOPO_RC_MASK) { case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED: if (mpssas_add_device(sc, - le16toh(phy->AttachedDevHandle), phy->LinkRate)){ - printf("%s: failed to add device with " - "handle 0x%x\n", __func__, + le16toh(phy->AttachedDevHandle), + phy->LinkRate)){ + mps_dprint(sc, MPS_ERROR, "%s: " + "failed to add device with handle " + "0x%x\n", __func__, le16toh(phy->AttachedDevHandle)); mpssas_prepare_remove(sassc, le16toh( phy->AttachedDevHandle)); @@ -283,8 +285,8 @@ element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0]; - id = mps_mapping_get_raid_id_from_handle - (sc, element->VolDevHandle); + id = mps_mapping_get_raid_tid_from_handle(sc, + element->VolDevHandle); mps_mapping_ir_config_change_event(sc, event_data); @@ -293,7 +295,8 @@ case MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED: case MPI2_EVENT_IR_CHANGE_RC_ADDED: if (!foreign_config) { - if (mpssas_volume_add(sc, le16toh(element->VolDevHandle))){ + if (mpssas_volume_add(sc, + le16toh(element->VolDevHandle))){ printf("%s: failed to add RAID " "volume with handle 0x%x\n", __func__, le16toh(element-> @@ -333,12 +336,16 @@ * Phys Disk of a volume has been created. Hide * it from the OS. */ - targ = mpssas_find_target_by_handle(sassc, 0, element->PhysDiskDevHandle); + targ = mpssas_find_target_by_handle(sassc, 0, + element->PhysDiskDevHandle); if (targ == NULL) break; - /* Set raid component flags only if it is not WD. - * OR WrapDrive with WD_HIDE_ALWAYS/WD_HIDE_IF_VOLUME is set in NVRAM + /* + * Set raid component flags only if it is not + * WD. OR WrapDrive with + * WD_HIDE_ALWAYS/WD_HIDE_IF_VOLUME is set in + * NVRAM */ if((!sc->WD_available) || ((sc->WD_available && @@ -674,10 +681,17 @@ * 1 - use the PhyNum field as a fallback to the mapping logic * 0 - never use the PhyNum field * -1 - only use the PhyNum field + * + * Note that using the Phy number to map a device can cause device adds + * to fail if multiple enclosures/expanders are in the topology. For + * example, if two devices are in the same slot number in two different + * enclosures within the topology, only one of those devices will be + * added. PhyNum mapping should not be used if multiple enclosures are + * in the topology. */ id = MPS_MAP_BAD_ID; if (sc->use_phynum != -1) - id = mps_mapping_get_sas_id(sc, sas_address, handle); + id = mps_mapping_get_tid(sc, sas_address, handle); if (id == MPS_MAP_BAD_ID) { if ((sc->use_phynum == 0) || ((id = config_page.PhyNum) > sassc->maxtargets)) { @@ -688,19 +702,31 @@ goto out; } } + mps_dprint(sc, MPS_MAPPING, "%s: Target ID for added device is %d.\n", + __func__, id); - if (mpssas_check_id(sassc, id) != 0) { - device_printf(sc->mps_dev, "Excluding target id %d\n", id); - error = ENXIO; - goto out; - } - + /* + * Only do the ID check and reuse check if the target is not from a + * RAID Component. For Physical Disks of a Volume, the ID will be reused + * when a volume is deleted because the mapping entry for the PD will + * still be in the mapping table. The ID check should not be done here + * either since this PD is already being used. + */ targ = &sassc->targets[id]; - if (targ->handle != 0x0) { - mps_dprint(sc, MPS_MAPPING, "Attempting to reuse target id " - "%d handle 0x%04x\n", id, targ->handle); - error = ENXIO; - goto out; + if (!(targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT)) { + if (mpssas_check_id(sassc, id) != 0) { + device_printf(sc->mps_dev, "Excluding target id %d\n", + id); + error = ENXIO; + goto out; + } + + if (targ->handle != 0x0) { + mps_dprint(sc, MPS_MAPPING, "Attempting to reuse " + "target id %d handle 0x%04x\n", id, targ->handle); + error = ENXIO; + goto out; + } } mps_dprint(sc, MPS_MAPPING, "SAS Address from SAS device page0 = %jx\n", @@ -787,7 +813,6 @@ } mpssas_startup_decrement(sassc); return (error); - } int @@ -1040,7 +1065,7 @@ goto out; } - id = mps_mapping_get_raid_id(sc, wwid, handle); + id = mps_mapping_get_raid_tid(sc, wwid, handle); if (id == MPS_MAP_BAD_ID) { printf("%s: could not get ID for volume with handle 0x%04x and " "WWID 0x%016llx\n", __func__, handle, @@ -1099,7 +1124,7 @@ */ sc->SSU_started = TRUE; sc->SSU_refcount = 0; - for (targetid = 0; targetid < sc->facts->MaxTargets; targetid++) { + for (targetid = 0; targetid < sc->max_devices; targetid++) { target = &sassc->targets[targetid]; if (target->handle == 0x0) { continue; @@ -1281,7 +1306,7 @@ * 3: enable to SSD and HDD * anything else will default to 1. */ - for (targetid = 0; targetid < sc->facts->MaxTargets; targetid++) { + for (targetid = 0; targetid < sc->max_devices; targetid++) { target = &sc->sassc->targets[targetid]; if (target->handle == 0x0) { continue; Index: sys/dev/mps/mps_user.c =================================================================== --- sys/dev/mps/mps_user.c +++ sys/dev/mps/mps_user.c @@ -2052,7 +2052,7 @@ data->DevHandle = dev_handle; } else { bus = 0; - target = mps_mapping_get_sas_id_from_handle(sc, dev_handle); + target = mps_mapping_get_tid_from_handle(sc, dev_handle); data->Bus = bus; data->TargetID = target; } Index: sys/dev/mps/mpsvar.h =================================================================== --- sys/dev/mps/mpsvar.h +++ sys/dev/mps/mpsvar.h @@ -33,7 +33,7 @@ #ifndef _MPSVAR_H #define _MPSVAR_H -#define MPS_DRIVER_VERSION "21.01.00.00-fbsd" +#define MPS_DRIVER_VERSION "21.02.00.00-fbsd" #define MPS_DB_MAX_WAIT 2500 @@ -53,6 +53,7 @@ #define MPS_PERIODIC_DELAY 1 /* 1 second heartbeat/watchdog check */ #define MPS_ATA_ID_TIMEOUT 5 /* 5 second timeout for SATA ID cmd */ +#define MPS_MISSING_CHECK_DELAY 10 /* 10 seconds between missing check */ #define MPS_SCSI_RI_INVALID_FRAME (0x00000002) #define MPS_STRING_LENGTH 64 @@ -70,7 +71,6 @@ #define MPS_MAX_MISSING_COUNT 0x0F #define MPS_DEV_RESERVED 0x20000000 #define MPS_MAP_IN_USE 0x10000000 -#define MPS_RAID_CHANNEL 1 #define MPS_MAP_BAD_ID 0xFFFFFFFF /* @@ -107,7 +107,6 @@ * @phy_bits: bitfields indicating controller phys * @dpm_entry_num: index of this device in device persistent map table * @dev_handle: device handle for the device pointed by this entry - * @channel: target channel * @id: target id * @missing_count: number of times the device not detected by driver * @hide_flag: Hide this physical disk/not (foreign configuration) @@ -119,8 +118,7 @@ u32 phy_bits; u16 dpm_entry_num; u16 dev_handle; - u8 reserved1; - u8 channel; + u16 reserved1; u16 id; u8 missing_count; u8 init_complete; @@ -293,6 +291,7 @@ struct mps_command *commands; struct mps_chain *chains; struct callout periodic; + struct callout device_check_callout; struct mpssas_softc *sassc; char tmp_string[MPS_STRING_LENGTH]; @@ -377,13 +376,10 @@ uint8_t max_volumes; uint8_t num_enc_table_entries; uint8_t num_rsvd_entries; - uint8_t num_channels; uint16_t max_dpm_entries; uint8_t is_dpm_enable; uint8_t track_mapping_events; uint32_t pending_map_events; - uint8_t mt_full_retry; - uint8_t mt_add_device_failed; /* FW diag Buffer List */ mps_fw_diagnostic_buffer_t @@ -732,19 +728,18 @@ int mps_mapping_initialize(struct mps_softc *); void mps_mapping_topology_change_event(struct mps_softc *, Mpi2EventDataSasTopologyChangeList_t *); -int mps_mapping_is_reinit_required(struct mps_softc *); void mps_mapping_free_memory(struct mps_softc *sc); int mps_config_set_dpm_pg0(struct mps_softc *, Mpi2ConfigReply_t *, Mpi2DriverMappingPage0_t *, u16 ); void mps_mapping_exit(struct mps_softc *); -void mps_mapping_check_devices(struct mps_softc *, int); +void mps_mapping_check_devices(void *); int mps_mapping_allocate_memory(struct mps_softc *sc); -unsigned int mps_mapping_get_sas_id(struct mps_softc *, uint64_t , u16); -unsigned int mps_mapping_get_sas_id_from_handle(struct mps_softc *sc, +unsigned int mps_mapping_get_tid(struct mps_softc *, uint64_t , u16); +unsigned int mps_mapping_get_tid_from_handle(struct mps_softc *sc, u16 handle); -unsigned int mps_mapping_get_raid_id(struct mps_softc *sc, u64 wwid, - u16 handle); -unsigned int mps_mapping_get_raid_id_from_handle(struct mps_softc *sc, +unsigned int mps_mapping_get_raid_tid(struct mps_softc *sc, u64 wwid, + u16 volHandle); +unsigned int mps_mapping_get_raid_tid_from_handle(struct mps_softc *sc, u16 volHandle); void mps_mapping_enclosure_dev_status_change_event(struct mps_softc *, Mpi2EventDataSasEnclDevStatusChange_t *event_data);