From d9e557459c004bebfea46a1b44de2fcdc1041c3d Mon Sep 17 00:00:00 2001 From: Ronny H <138828701+ron-unstructured@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:48:42 -0700 Subject: [PATCH] Update link_urls metadata (#2646) Update the metadata `link_urls` in the News-of-the-Day notebook example. --- .../news-of-the-day.ipynb | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/examples/chroma-news-of-the-day/news-of-the-day.ipynb b/examples/chroma-news-of-the-day/news-of-the-day.ipynb index b9cc605cc..cee6db7d5 100644 --- a/examples/chroma-news-of-the-day/news-of-the-day.ipynb +++ b/examples/chroma-news-of-the-day/news-of-the-day.ipynb @@ -58,10 +58,11 @@ "outputs": [], "source": [ "links = []\n", + "\n", "for element in elements:\n", - " if element.metadata.links is not None:\n", - " relative_link = element.metadata.links[0][\"url\"][1:]\n", - " if relative_link.startswith(\"2023\"):\n", + " if element.metadata.link_urls:\n", + " relative_link = element.metadata.link_urls[0][1:]\n", + " if relative_link.startswith(\"2024\"):\n", " links.append(f\"{cnn_lite_url}{relative_link}\")" ] }, @@ -74,11 +75,7 @@ { "data": { "text/plain": [ - "['https://lite.cnn.com/2023/08/07/health/breast-cancer-overdiagnosis/index.html',\n", - " 'https://lite.cnn.com/2023/08/08/investing/ups-earnings/index.html',\n", - " 'https://lite.cnn.com/2023/08/08/business/molson-coors-blue-run-spirits-acquisition/index.html',\n", - " 'https://lite.cnn.com/2023/08/08/politics/ukraine-counteroffensive-us-briefings/index.html',\n", - " 'https://lite.cnn.com/2023/08/08/europe/italian-cheesemaker-dies-italy-intl-scli/index.html']" + "98" ] }, "execution_count": 5, @@ -87,7 +84,7 @@ } ], "source": [ - "links[:5]" + "len(links)" ] }, { @@ -109,7 +106,7 @@ "source": [ "from langchain.document_loaders import UnstructuredURLLoader\n", "\n", - "loaders = UnstructuredURLLoader(urls=links, show_progress_bar=True)" + "loaders = UnstructuredURLLoader(urls=links[:20], show_progress_bar=True)" ] }, { @@ -122,7 +119,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 97/97 [00:11<00:00, 8.41it/s]\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 20/20 [00:04<00:00, 4.67it/s]\n" ] } ], @@ -139,7 +136,7 @@ { "data": { "text/plain": [ - "Document(page_content='CNN\\n\\n8/8/2023\\n\\nOlder women’s breast cancer is often overdiagnosed, study finds, raising risk of unnecessary treatment\\n\\nBy Amanda Musa, CNN\\n\\nUpdated: \\n 8:38 AM EDT, Tue August 8, 2023\\n\\nSource: CNN\\n\\nA breast cancer diagnosis is an all-too-common reality for women around the world. In the US, about 240,000 cases of breast cancer are diagnosed in women every year, the US Centers for Disease Control and Prevention estimates.\\n\\nHealth care providers and patients alike are usually inclined to pursue treatment to stop the disease. But some experts say that it isn’t always necessary to treat breast cancer in older women with aggressive therapy.\\n\\nA study published Monday in the Annals of Internal Medicine found that large numbers of American women ages 70 to 85 are potentially overdiagnosed with breast cancer and therefore could receive unnecessary treatment.\\n\\n“Overdiagnosis refers to this phenomenon where we find breast cancers on screening that never would have become clinically apparent,” says Dr. Ilana Richman, the lead author of the study and an assistant professor of medicine at the Yale School of Medicine.\\n\\n“The breast cancers are real, true cancers under the microscope, but they would have laid dormant in a person’s body and never caused symptoms,” she added. “We wouldn’t have known about that had we not gone looking.”\\n\\nThe study looked at 54,635 women 70 or older who had recently been screened for breast cancer. Richman and her team estimate that 31% of women ages 70 to 74 were potentially overdiagnosed. Among women 75 to 84, an estimated 47% were potentially overdiagnosed.\\n\\nWomen 85 and older were the age group most likely to be overdiagnosed, at 54%.\\n\\nBreast cancer diagnosis peaks among women in their 70s, according to data from the American Cancer Society. The risk decreases as women age into their 80s, partly “because women die of other health conditions instead like heart disease or other cancers,” Richman says.\\n\\nWhether an older women should get screened for breast cancer is an individual decision, said Richman, whose study found that overdiagnosis of breast cancer was more prevalent as age increased and life expectancy decreased.\\n\\nTechnological developments drive overdiagnosis\\n\\nMore women are being diagnosed with breast cancer thanks to advancements in screening such as three-dimensional mammography as well as CT, MRI and PET scanning. Technology makes it easier for oncologists to find even the smallest\\xa0cancer, said Dr. Otis Brawley, an oncologist and epidemiologist at Johns Hopkins University.\\n\\n“We can now find a lesion in a woman’s breasts that is the size of a green pea,” he said. “We can stick a needle into that woman’s breast and get the needle into that green pea-sized lesion and send a piece of that green pea-sized lesion to a pathologist.”\\n\\nBut Brawley notes that some of these breast cancers are not destined to grow, spread or kill due to their genetic makeup – and meeting them with aggressive treatment such as surgery, radiation or chemotherapy can sometimes be unnecessary and even harmful to the patient.\\n\\nNot only can overtreatment increase the risk of complications, especially in older patients, it can cause unwarranted stress and financial trouble, he said.\\n\\n“There’s also the experience of being diagnosed,” Richman says. “It is one of the most life-changing experiences. If we could spare women that experience who don’t need to have it, then I think that would be beneficial.”\\n\\nShe doesn’t want to suggest that less-threatening cancers always go untreated, however. Less-invasive surgery and the use of medications that block estrogen can often treat these cancers sufficiently without exposing patients to radiation or chemotherapy.\\n\\nMammograms and older women\\n\\nFor women over the age of 75, there is no guidance on whether to continue getting mammograms, according to the US Preventive Services Task Force, a group of independent medical experts whose recommendations help guide doctors’ decisions. The evidence determining the risks and benefits of screening in women in this age group is “insufficient,” the group says.\\n\\nIn May, the USPSTF issued a draft recommendation urging women to start breast cancer screenings at age 40. It also noted an effort to conduct more research on whether women 75 and older need to be screened for breast cancer.\\n\\nMammograms for this age group cost the federal health plan for seniors more than\\xa0$410 million a year, according to a 2013 study in JAMA Internal Medicine.\\n\\nTaxpayers usually foot the bill for these tests because most seniors are covered by Medicare.\\n\\nAnd while cancer screenings generally aren’t expensive – a\\xa0mammogram averages about $100\\xa0– they can launch a cascade of follow-up tests and treatments that add to the total cost of care.\\n\\nMuch of the screening guidance for mammograms also depends on how much longer a person is expected to live, Richman says, something that’s hard to quantify and a topic that most patients and doctors are not eager to discuss.\\n\\n“For any one individual in front of you, there’s a fair bit of uncertainty about how long that person will live,” she said. “It also raises issues that are just typical to talk about, like, certainly nobody wants to hear that they’re not going to live long enough to benefit” from a mammogram.\\n\\nRichman adds that more research needs to be done to conclude whether a woman of a certain age needs to get a mammogram in the first place.\\n\\n“For women who are getting into the upper age range or women who have lots of other serious medical conditions that are also taking up a lot of time and deserve attention, it’s unlikely that mammography is going to be the thing that is most beneficial to that person’s health.”\\n\\nRichman also worries about unnecessary exposure to radiation during a mammogram.\\n\\nUltimately, she says, the decision to get screened is up to the individual.\\n\\n“I think for women who are used to being screened and feel like they derive value from knowing something about their bodies, that’s an important consideration,” she says.\\n\\nThe future of breast cancer diagnosis\\n\\nIn the near future, doctors will be able to monitor “good-behaving” breast cancers in the same way low-risk prostate cancers are treated, Brawley said, adding that estimates say “overdiagnosis occurs in more than half of all screen-detected prostate cancer.”\\n\\n“Within the next 10 years, there will be some women who are going to be told ‘you have breast cancer, but it is one of the more benign breast cancers. Therefore, we’re going to watch you instead of treating you aggressively,”\\xa0he said.\\n\\nUltimately, Brawley says, the answer to overdiagnosis is the study of cancer genomics, analyzing the molecular biology of an individual’s cancer. But the technology is not quite there yet.\\n\\n“It will lead to a redefinition of cancer. The pathologic definition used today was developed in the mid-19th century using the biopsy and a light microscope. We need to move to a 21st-century definition, encompassing both the biopsy and pathologic appearance as well as genomics,” Brawley says. “The 21st-century definition of cancer will recognize that breast cancer is not one entity but many distinct diseases with potentially different patterns of behavior calling for different treatments and, sometimes, no treatment.”\\n\\nCNN’s Jacqueline Howard contributed to this report.\\n\\nCorrection: A previous version of this story misspelled Dr. Ilana Richman’s name.\\n\\nSee Full Web Article\\n\\nGo to the full CNN experience\\n\\n© 2023 Cable News Network. A Warner Bros. Discovery Company. All Rights Reserved.\\n\\nTerms of Use\\n\\nPrivacy Policy\\n\\nAd Choices\\n\\nCookie Settings', metadata={'source': 'https://lite.cnn.com/2023/08/07/health/breast-cancer-overdiagnosis/index.html'})" + "Document(page_content='CNN\\n\\n3/13/2024\\n\\nRFK Jr.’s VP prospect Aaron Rodgers has shared\\xa0false\\xa0Sandy Hook conspiracy theories\\xa0in private conversations\\n\\nBy Pamela Brown and Jake Tapper, CNN\\n\\nUpdated: \\n 5:33 PM EDT, Wed March 13, 2024\\n\\nSource: CNN\\n\\nIndependent presidential candidate Robert F. Kennedy Jr. has confirmed that among his potential vice-presidential prospects is New York Jets quarterback Aaron Rodgers,\\xa0who\\xa0in private conversations shared deranged conspiracy theories\\xa0about the 2012 Sandy Hook school shooting not being real.\\n\\nCNN knows of two people with whom Rodgers has enthusiastically shared these stories,\\xa0including with Pamela Brown, one of the journalists writing this piece.\\n\\nBrown was covering the Kentucky Derby for CNN in\\xa02013\\xa0when she was introduced to Rodgers, then with the Green Bay Packers, at a post-Derby party. Hearing that she was a journalist with CNN, Rodgers immediately began attacking the news media for covering up important stories. Rodgers brought up the tragic killing\\xa0of 20 children and 6 adults by a gunman at Sandy Hook Elementary School,\\xa0claiming\\xa0it was actually a government inside job and the media was intentionally ignoring it.\\n\\nWhen Brown questioned him on the evidence to show this very real shooting was staged, Rodgers began sharing various theories that have been disproven numerous times. Such conspiracy theories were also later at the center of lawsuits brought by victims’ families when they sued conspiracy theorist Alex Jones on the matter.\\n\\nJones baselessly repeating lies that the 2012 mass shooting was staged, and that the families and first responders were “crisis actors,” spawned multiple lawsuits and a trial was held in 2022 over lawsuits that were filed in Connecticut.\\n\\nFamily members throughout that trial described in poignant terms how the lies had prompted unrelenting harassment against them and compounded the emotional agony of losing their loved ones.\\n\\nBrown recalls Rodgers asking her if she thought it was off that there were men in black in the woods by the school,\\xa0falsely\\xa0claiming those men were actually government operatives. Brown found the encounter disturbing.\\n\\nRodgers, through one of his agents, declined to comment to CNN.\\n\\nCNN has spoken to another person with a similar story. This person, to whom CNN has granted anonymity so as to avoid harassment, recalled that several years ago, Rodgers claimed, “Sandy Hook never happened…All those children never existed. They were all actors.”\\n\\nWhen asked about the grieving parents, the source recalled Rodgers saying, “They’re all making it up. They’re all actors.”\\n\\nRodgers went on to delve into some of the darker caverns of the false conspiracy theory. This person found the encounter disturbing.\\n\\nKennedy’s campaign\\xa0announced Wednesday that the candidate will name his running mate on March 26 in Oakland, California. Kennedy told CNN in an interview on Tuesday that he had recently met with Rodgers, as well as former Minnesota Gov. Jesse Ventura, about the possibility of joining his campaign.\\n\\nSee Full Web Article\\n\\nGo to the full CNN experience\\n\\n© 2024 Cable News Network. A Warner Bros. Discovery Company. All Rights Reserved.\\n\\nTerms of Use\\n\\nPrivacy Policy\\n\\nAd Choices\\n\\nCookie Settings', metadata={'source': 'https://lite.cnn.com/2024/03/13/politics/aaron-rodgers-sandy-hook-conspiracy-theories/index.html'})" ] }, "execution_count": 8, @@ -190,7 +187,7 @@ "metadata": {}, "outputs": [], "source": [ - "query_docs = vectorstore.similarity_search(\"Update on the coup in Niger.\", k=1)" + "query_docs = vectorstore.similarity_search(\"What is behind the rapid increase in car insurance rates?\", k=1)" ] }, { @@ -235,13 +232,21 @@ "name": "stdout", "output_type": "stream", "text": [ - "Niger's military has deployed reinforcements to the capital city after refusing to cede power following a deadline set by a regional bloc. The military junta took control of the country in a coup last month, leading to political chaos. The Economic Community of West African States (ECOWAS) has imposed sanctions and threatened military intervention if the junta does not step down. The situation remains uncertain, with ECOWAS leaders seeking a diplomatic solution but willing to use force as a last resort. The uncertainty has caused concern among residents, who are stocking up on supplies and attempting to flee the capital. The future of Niger's elected government is important to its democratic neighbors and Western partners, including the United States and France. Russia's mercenary group Wagner has also shown interest in the situation, potentially seeking to gain influence in the region.\n" + "Car insurance rates in the US have increased by almost 21% in the past year, contributing to the overall inflation rate. The rise can be attributed to rising car repair costs, more severe and frequent car accidents, and riskier driving behaviors. The increase in rates varies by state, with Nevada experiencing the highest jump and North Carolina the smallest. While rates are expected to moderate nationally in the second half of 2024, some markets may continue to see increases.\n" ] } ], "source": [ "print(chain.run(query_docs))" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "283ed440-042c-4a84-8956-54a4912d9da6", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -260,7 +265,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4,